From bf9b5e618800d3ce9007396a9b4d2cc073077c46 Mon Sep 17 00:00:00 2001 From: David Beck Date: Fri, 17 Apr 2026 23:27:57 +0100 Subject: [PATCH] Stats collection for compressed chunks This change collects information about compressed chunks and stores them in an LRU fashion. It captures events at compression, query and DML oparations. The query/DML operations maintain totals and information about the last operation. The chunk information is maintained in shared memory per database. One can access the data via calling the - `_timescaledb_functions.chunk_statistics(..)` function or by selecting from the - `timescaledb_information.stat_chunk_activity` view the former allows the user to push down filters, so it is faster and with less lock contention. The amount of shared memory is controlled by the GUC: - `timescaledb.stats_max_chunks` If the GUC is set to zero, the feature is disabled. The default value is 1024 and the maximum is 262144. The amount must be a power of two number. The `stats_max_chunks` value controls how many chunk we store information about. The memory consumption is 300 bytes per entry. If the loader is not upgraded, the statistics will be disabled on PG 15/16. PG 17 doesn't require a loader change, so we can upgrade the extension without restarting: Disable-check: loader-change Co-authored-by: Ante Kresic Signed-off-by: David Beck --- .../workflows/linux-32bit-build-and-test.yaml | 1 + .github/workflows/windows-build-and-test.yaml | 1 + .unreleased/pr_9730 | 1 + cmake/ScriptFiles.cmake | 1 + sql/ts_stats.sql | 71 + sql/updates/reverse-dev.sql | 5 + sql/views.sql | 69 + src/CMakeLists.txt | 1 + src/chunk.c | 4 + src/guc.c | 57 + src/guc.h | 7 + src/loader/CMakeLists.txt | 1 + src/loader/loader.c | 7 + src/loader/ts_stats_handles.c | 65 + src/loader/ts_stats_handles.h | 41 + src/ts_stats/CMakeLists.txt | 5 + src/ts_stats/ts_stats_defs.h | 370 +++++ src/ts_stats/ts_stats_record.c | 338 ++++ src/ts_stats/ts_stats_record.h | 78 + src/ts_stats/ts_stats_segment.c | 685 ++++++++ src/ts_stats/ts_stats_segment.h | 55 + src/ts_stats/ts_stats_srf.c | 437 ++++++ test/expected/c_unit_tests.out | 8 + test/expected/observ_funcs.out | 55 + test/expected/pg_dump.out | 1 + test/sql/CMakeLists.txt | 1 + test/sql/c_unit_tests.sql | 4 + test/sql/observ_funcs.sql | 26 + test/src/CMakeLists.txt | 1 + test/src/test_stats.c | 139 ++ tsl/src/chunk_split.c | 6 +- tsl/src/compression/compression.c | 46 +- tsl/src/compression/compression.h | 16 +- tsl/src/compression/compression_dml.c | 61 +- tsl/src/compression/recompress.c | 8 +- .../nodes/columnar_scan/compressed_batch.c | 9 + .../nodes/columnar_scan/decompress_context.h | 7 + tsl/src/nodes/columnar_scan/exec.c | 17 + tsl/test/expected/compress_observ-15.out | 1395 +++++++++++++++++ tsl/test/expected/compress_observ-16.out | 1395 +++++++++++++++++ tsl/test/expected/compress_observ-17.out | 1395 +++++++++++++++++ tsl/test/expected/compress_observ-18.out | 1395 +++++++++++++++++ tsl/test/shared/expected/extension.out | 2 + tsl/test/sql/.gitignore | 1 + tsl/test/sql/CMakeLists.txt | 1 + tsl/test/sql/compress_observ.sql.in | 202 +++ 46 files changed, 8478 insertions(+), 13 deletions(-) create mode 100644 .unreleased/pr_9730 create mode 100644 sql/ts_stats.sql create mode 100644 src/loader/ts_stats_handles.c create mode 100644 src/loader/ts_stats_handles.h create mode 100644 src/ts_stats/CMakeLists.txt create mode 100644 src/ts_stats/ts_stats_defs.h create mode 100644 src/ts_stats/ts_stats_record.c create mode 100644 src/ts_stats/ts_stats_record.h create mode 100644 src/ts_stats/ts_stats_segment.c create mode 100644 src/ts_stats/ts_stats_segment.h create mode 100644 src/ts_stats/ts_stats_srf.c create mode 100644 test/expected/observ_funcs.out create mode 100644 test/sql/observ_funcs.sql create mode 100644 test/src/test_stats.c create mode 100644 tsl/test/expected/compress_observ-15.out create mode 100644 tsl/test/expected/compress_observ-16.out create mode 100644 tsl/test/expected/compress_observ-17.out create mode 100644 tsl/test/expected/compress_observ-18.out create mode 100644 tsl/test/sql/compress_observ.sql.in diff --git a/.github/workflows/linux-32bit-build-and-test.yaml b/.github/workflows/linux-32bit-build-and-test.yaml index 571cca97ff5..f083efaeee4 100644 --- a/.github/workflows/linux-32bit-build-and-test.yaml +++ b/.github/workflows/linux-32bit-build-and-test.yaml @@ -70,6 +70,7 @@ jobs: compress_bloom_sparse_debug compress_compbloom_hash_pushdown compress_composite_bloom_debug + compress_observ-* compress_qualpushdown_saop compress_sort_transform compression_allocation diff --git a/.github/workflows/windows-build-and-test.yaml b/.github/workflows/windows-build-and-test.yaml index fb9396e426c..102e071d7a8 100644 --- a/.github/workflows/windows-build-and-test.yaml +++ b/.github/workflows/windows-build-and-test.yaml @@ -89,6 +89,7 @@ jobs: compress_bloom_sparse_debug compress_compbloom_hash_pushdown compress_composite_bloom_debug + compress_observ-* compress_qualpushdown_saop compress_sort_transform compression_algos diff --git a/.unreleased/pr_9730 b/.unreleased/pr_9730 new file mode 100644 index 00000000000..b183b7a8174 --- /dev/null +++ b/.unreleased/pr_9730 @@ -0,0 +1 @@ +Implements: #9730 In memory observability for compressed chunks diff --git a/cmake/ScriptFiles.cmake b/cmake/ScriptFiles.cmake index 06b20556a34..d8df1359fee 100644 --- a/cmake/ScriptFiles.cmake +++ b/cmake/ScriptFiles.cmake @@ -42,6 +42,7 @@ set(SOURCE_FILES bgw_scheduler.sql metadata.sql uuidv7.sql + ts_stats.sql views.sql views_experimental.sql gapfill.sql diff --git a/sql/ts_stats.sql b/sql/ts_stats.sql new file mode 100644 index 00000000000..7e7410404e3 --- /dev/null +++ b/sql/ts_stats.sql @@ -0,0 +1,71 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +-- Chunk statistics. This function takes three optional parameters: +-- - compressed_relid: if provided, returns stats for the specified compressed chunk only; otherwise, returns stats for all chunks. +-- - uncompressed_relid: if provided, returns stats for the specified user chunk only; if compressed_relid is also provided, both must agree. +-- - since: if provided, returns stats for operations that occurred since the specified timestamp; otherwise, returns all stats. +-- +CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_statistics( + compressed_relid regclass DEFAULT NULL, + uncompressed_relid regclass DEFAULT NULL, + since timestamptz DEFAULT NULL +) +RETURNS TABLE ( + compressed_relid oid, + uncompressed_relid oid, + -- Compression + compressed_batch_count bigint, + compressed_block_count bigint, + compressed_batch_rows_min bigint, + compressed_batch_rows_max bigint, + compressed_batch_rows_sum bigint, + compressed_batch_rows_sqsum double precision, + compressed_batch_bytes_min bigint, + compressed_batch_bytes_max bigint, + compressed_batch_bytes_sum bigint, + compressed_batch_bytes_sqsum double precision, + compressed_block_bytes_min bigint, + compressed_block_bytes_max bigint, + compressed_block_bytes_sum bigint, + compressed_block_bytes_sqsum double precision, + -- DML totals (cumulative, from cmd_totals) + total_batches_deleted bigint, + total_batches_decompressed bigint, + total_tuples_decompressed bigint, + total_batches_scanned bigint, + total_batches_checked_by_bloom bigint, + total_batches_pruned_by_bloom bigint, + total_batches_without_bloom bigint, + total_batches_bloom_false_positives bigint, + total_batches_filtered_compressed bigint, + total_batches_filtered_decompressed bigint, + -- DML last operation snapshot (from cmd_last_op) + last_op_batches_deleted bigint, + last_op_batches_decompressed bigint, + last_op_tuples_decompressed bigint, + last_op_batches_scanned bigint, + last_op_batches_checked_by_bloom bigint, + last_op_batches_pruned_by_bloom bigint, + last_op_batches_without_bloom bigint, + last_op_batches_bloom_false_positives bigint, + last_op_batches_filtered_compressed bigint, + last_op_batches_filtered_decompressed bigint, + -- Operation counts + n_selects bigint, + n_inserts bigint, + n_updates bigint, + n_deletes bigint, + -- Timestamps + first_update timestamptz, + last_update timestamptz +) +AS '@MODULE_PATHNAME@', 'ts_stats_chunks' +LANGUAGE C STABLE; + +-- Logical reset: clears all cached chunks under the segment lock. +CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_statistics_reset() +RETURNS VOID +AS '@MODULE_PATHNAME@', 'ts_stats_reset' +LANGUAGE C VOLATILE; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index 3ad68d7015c..ff5bccc0788 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -10,6 +10,11 @@ JOIN pg_constraint con WHERE c.osm_chunk ON CONFLICT DO NOTHING; +-- Drop chunk stats related objects +DROP VIEW IF EXISTS timescaledb_information.stat_chunk_activity; +DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_statistics(regclass, regclass, timestamptz); +DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_statistics_reset(); + -- Restore chunk_constraint rows for outbound FKs by matching chunk-side -- FKs to their hypertable-side counterpart by name. INSERT INTO _timescaledb_catalog.chunk_constraint diff --git a/sql/views.sql b/sql/views.sql index 3fbcaec8cf7..7fe64322f3b 100644 --- a/sql/views.sql +++ b/sql/views.sql @@ -430,5 +430,74 @@ SELECT * FROM timescaledb_information.chunk_compression_settings; CREATE OR REPLACE VIEW _timescaledb_config.bgw_job AS SELECT * from _timescaledb_catalog.bgw_job; +-- chunk statistics view +CREATE OR REPLACE VIEW timescaledb_information.stat_chunk_activity AS +SELECT + o.uncompressed_relid::regclass AS chunk, + o.compressed_relid::regclass AS compressed_chunk, + c.id AS chunk_id, + c.hypertable_id, + h.table_name AS hypertable, + -- Compression + o.compressed_batch_count, + o.compressed_block_count, + o.compressed_batch_rows_min, + o.compressed_batch_rows_max, + o.compressed_batch_rows_sum / NULLIF(o.compressed_batch_count, 0) + AS compressed_batch_rows_avg, + sqrt(o.compressed_batch_rows_sqsum / NULLIF(o.compressed_batch_count, 0) + - power(o.compressed_batch_rows_sum::double precision + / NULLIF(o.compressed_batch_count, 0), 2)) + AS compressed_batch_rows_stddev, + o.compressed_batch_bytes_min, + o.compressed_batch_bytes_max, + o.compressed_batch_bytes_sum / NULLIF(o.compressed_batch_count, 0) + AS compressed_batch_bytes_avg, + sqrt(o.compressed_batch_bytes_sqsum / NULLIF(o.compressed_batch_count, 0) + - power(o.compressed_batch_bytes_sum::double precision + / NULLIF(o.compressed_batch_count, 0), 2)) + AS compressed_batch_bytes_stddev, + o.compressed_block_bytes_min, + o.compressed_block_bytes_max, + o.compressed_block_bytes_sum / NULLIF(o.compressed_block_count, 0) + AS compressed_block_bytes_avg, + sqrt(o.compressed_block_bytes_sqsum / NULLIF(o.compressed_block_count, 0) + - power(o.compressed_block_bytes_sum::double precision + / NULLIF(o.compressed_block_count, 0), 2)) + AS compressed_block_bytes_stddev, + -- CMD totals + o.total_batches_deleted, + o.total_batches_decompressed, + o.total_tuples_decompressed, + o.total_batches_scanned, + o.total_batches_checked_by_bloom, + o.total_batches_pruned_by_bloom, + o.total_batches_without_bloom, + o.total_batches_bloom_false_positives, + o.total_batches_filtered_compressed, + o.total_batches_filtered_decompressed, + -- last CMD operation + o.last_op_batches_deleted, + o.last_op_batches_decompressed, + o.last_op_tuples_decompressed, + o.last_op_batches_scanned, + o.last_op_batches_checked_by_bloom, + o.last_op_batches_pruned_by_bloom, + o.last_op_batches_without_bloom, + o.last_op_batches_bloom_false_positives, + o.last_op_batches_filtered_compressed, + o.last_op_batches_filtered_decompressed, + o.n_selects, + o.n_inserts, + o.n_updates, + o.n_deletes, + o.first_update, + o.last_update +FROM _timescaledb_functions.chunk_statistics() o +LEFT JOIN _timescaledb_catalog.chunk c + ON format('%I.%I', c.schema_name, c.table_name)::regclass = o.uncompressed_relid +LEFT JOIN _timescaledb_catalog.hypertable h ON h.id = c.hypertable_id; + + GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_config TO PUBLIC; GRANT SELECT ON ALL TABLES IN SCHEMA timescaledb_information TO PUBLIC; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7f1640f1db9..29e03ff8287 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -125,6 +125,7 @@ add_subdirectory(ts_catalog) add_subdirectory(nodes) add_subdirectory(planner) add_subdirectory(with_clause) +add_subdirectory(ts_stats) # Don't run clang-tidy on the files we copied from Postgres. We don't want to # introduce changes there unless absolutely necessary. CMake can only access the diff --git a/src/chunk.c b/src/chunk.c index 7ba519e68cb..b794c75f37d 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -4074,6 +4075,9 @@ ts_chunk_drop(const Chunk *chunk, DropBehavior behavior, int32 log_level) /* Drop the table */ performDeletion(&objaddr, behavior, 0); + + /* Evict the chunk stats from the shared memory */ + ts_stats_chunk_evict(chunk->table_id); } static void diff --git a/src/guc.c b/src/guc.c index 8f23e2d1814..e315b269a3d 100644 --- a/src/guc.c +++ b/src/guc.c @@ -154,6 +154,8 @@ TSDLLEXPORT bool ts_guc_enable_delete_after_compression = false; TSDLLEXPORT bool ts_guc_enable_merge_on_cagg_refresh = false; bool ts_guc_enable_partitioned_hypertables = false; +TSDLLEXPORT int ts_guc_stats_max_chunks = TS_STATS_MAX_CHUNKS_DEFAULT; + #if PG16_GE TSDLLEXPORT bool ts_guc_enable_cagg_rewrites = false; TSDLLEXPORT bool ts_guc_cagg_rewrites_debug_info = false; @@ -494,6 +496,46 @@ assign_default_chunk_time_interval(const char *newval, void *extra) default_chunk_time_interval = extra; } +/* + * check_hook: accept 0 (feature disabled) or a power of two in [MIN, MAX]. + * Non-power-of-two values are rejected with a hint listing valid sizes. + */ +static bool +stats_max_chunks_check_hook(int *newval, void **extra, GucSource source) +{ + int v = *newval; + + if (v == 0) + { + return true; /* 0 = disabled, explicitly allowed */ + } + + if (v < TS_STATS_MAX_CHUNKS_MIN) + { + GUC_check_errdetail("Minimum cache capacity is %d chunks. " + "Set to 0 to disable the observability feature.", + TS_STATS_MAX_CHUNKS_MIN); + return false; + } + + if (v > TS_STATS_MAX_CHUNKS_MAX) + { + GUC_check_errdetail("Maximum cache capacity is %d chunks " + "(largest power of 2 within the slot index space).", + TS_STATS_MAX_CHUNKS_MAX); + return false; + } + + if ((v & (v - 1)) != 0) + { + GUC_check_errdetail("timescaledb.stats_max_chunks must be 0 (disabled) or a power of 2."); + GUC_check_errhint("Valid values: 0, 256, 512, 1024, 2048, " + "4096, 8192, 16384, 32768, 65536, 131072, 262144."); + return false; + } + return true; +} + void _guc_init(void) { @@ -1443,6 +1485,21 @@ _guc_init(void) NULL); #endif + DefineCustomIntVariable(MAKE_EXTOPTION("stats_max_chunks"), + "Per-database statistics cache capacity, " + "in chunks. 0 disables the feature.", + "Must be 0 or a power of 2. " + "Takes effect only after server restart.", + &ts_guc_stats_max_chunks, + TS_STATS_MAX_CHUNKS_DEFAULT, /* default */ + 0, /* min: 0 = disabled */ + TS_STATS_MAX_CHUNKS_MAX, /* max: 2^18 = 262144 chunks */ + PGC_SIGHUP, + 0, + stats_max_chunks_check_hook, + NULL, + NULL); + #ifdef USE_TELEMETRY DefineCustomEnumVariable(MAKE_EXTOPTION("telemetry_level"), "Telemetry settings level", diff --git a/src/guc.h b/src/guc.h index ce6f4c9bdd1..a843a201817 100644 --- a/src/guc.h +++ b/src/guc.h @@ -161,6 +161,13 @@ extern TSDLLEXPORT bool ts_guc_enable_rowlevel_compression_locking; extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_batch_sorted_merge; extern bool ts_guc_enable_partitioned_hypertables; +extern TSDLLEXPORT int ts_guc_stats_max_chunks; + +#define TS_STATS_MAX_CHUNKS_DEFAULT 1024 +#define TS_STATS_MAX_CHUNKS_MIN 256 +#define TS_STATS_MAX_CHUNKS_MAX 262144 + +#define IS_STATS_CHUNKS_ENABLED() (ts_guc_stats_max_chunks > 0) void _guc_init(void); diff --git a/src/loader/CMakeLists.txt b/src/loader/CMakeLists.txt index aad5071edc3..713870d3198 100644 --- a/src/loader/CMakeLists.txt +++ b/src/loader/CMakeLists.txt @@ -5,6 +5,7 @@ set(SOURCES bgw_launcher.c bgw_interface.c function_telemetry.c + ts_stats_handles.c lwlocks.c) set(TEST_SOURCES ${PROJECT_SOURCE_DIR}/test/src/symbol_conflict.c) diff --git a/src/loader/loader.c b/src/loader/loader.c index b7ef2f87420..f0b117429a4 100644 --- a/src/loader/loader.c +++ b/src/loader/loader.c @@ -41,6 +41,7 @@ #include "loader/function_telemetry.h" #include "loader/loader.h" #include "loader/lwlocks.h" +#include "loader/ts_stats_handles.h" /* * Loading process: @@ -623,6 +624,9 @@ timescaledb_shmem_startup_hook(void) ts_bgw_message_queue_shmem_startup(); ts_lwlocks_shmem_startup(); ts_function_telemetry_shmem_startup(); +#if PG17_LT + ts_stats_shmem_startup(); +#endif } /* @@ -642,6 +646,9 @@ timescaledb_shmem_request_hook(void) ts_bgw_message_queue_alloc(); ts_lwlocks_shmem_alloc(); ts_function_telemetry_shmem_alloc(); +#if PG17_LT + ts_stats_shmem_request(); +#endif } static void diff --git a/src/loader/ts_stats_handles.c b/src/loader/ts_stats_handles.c new file mode 100644 index 00000000000..9035da338a8 --- /dev/null +++ b/src/loader/ts_stats_handles.c @@ -0,0 +1,65 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#include "ts_stats_handles.h" + +#include +#include +#include +#include +#include + +/* + * This is the fallback implementation for PG 15-16 of the per-database shared memory + * segment for chunk observability. In PG 17+ we can use the DSM registry and avoid this complexity. + * The fallback implementation uses a rendezvous-based registry of DSM segments keyed by database + * OID. + */ + +#if PG17_LT + +static Size +ts_stats_handle_table_size(void) +{ + return MAXALIGN(sizeof(TsObservHandleTable)); +} + +void +ts_stats_shmem_request(void) +{ + RequestAddinShmemSpace(ts_stats_handle_table_size()); +} + +void +ts_stats_shmem_startup(void) +{ + bool found; + + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + + TsObservHandleTable *tbl = + ShmemInitStruct("ts_stats_handles", ts_stats_handle_table_size(), &found); + if (!found) + { + memset(tbl, 0, ts_stats_handle_table_size()); + LWLockInitialize(&tbl->lock, LWLockNewTrancheId()); + LWLockRegisterTranche(tbl->lock.tranche, "ts_stats_handles"); + tbl->max_entries = TS_STATS_MAX_DATABASES; + for (int i = 0; i < TS_STATS_MAX_DATABASES; i++) + { + tbl->entries[i].dboid = InvalidOid; + } + } + + LWLockRelease(AddinShmemInitLock); + + /* Publish via rendezvous for the versioned extension to find. */ + TsObservHandleTable **rv = + (TsObservHandleTable **) find_rendezvous_variable("ts_stats_handles"); + *rv = tbl; +} + +#endif /* PG17_LT */ diff --git a/src/loader/ts_stats_handles.h b/src/loader/ts_stats_handles.h new file mode 100644 index 00000000000..5fcabf42300 --- /dev/null +++ b/src/loader/ts_stats_handles.h @@ -0,0 +1,41 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#pragma once + +#include +#include +#include +#include + +/* + * See more details in ts_stats_segment.c these declaration are only for the fallback + * implementation of the per-database DSM segment for PG 15-16. + */ + +#if PG17_LT +#define TS_STATS_MAX_DATABASES 1024 + +/* Segment types */ +#define TS_STATS_TYPE_CHUNK_SEGMENT 1 + +typedef struct TsObservHandleEntry +{ + Oid dboid; + int segment_type; + dsm_handle handle; +} TsObservHandleEntry; + +typedef struct TsObservHandleTable +{ + LWLock lock; + uint32 max_entries; + TsObservHandleEntry entries[TS_STATS_MAX_DATABASES]; +} TsObservHandleTable; + +void ts_stats_shmem_request(void); +void ts_stats_shmem_startup(void); +#endif /* PG17_LT */ diff --git a/src/ts_stats/CMakeLists.txt b/src/ts_stats/CMakeLists.txt new file mode 100644 index 00000000000..51aa44c5a96 --- /dev/null +++ b/src/ts_stats/CMakeLists.txt @@ -0,0 +1,5 @@ +set(SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/ts_stats_record.c + ${CMAKE_CURRENT_SOURCE_DIR}/ts_stats_segment.c + ${CMAKE_CURRENT_SOURCE_DIR}/ts_stats_srf.c) +target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) diff --git a/src/ts_stats/ts_stats_defs.h b/src/ts_stats/ts_stats_defs.h new file mode 100644 index 00000000000..f11f3461a6f --- /dev/null +++ b/src/ts_stats/ts_stats_defs.h @@ -0,0 +1,370 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ +#pragma once + +#include +#include +#include "export.h" +#include +#include +#include +#include +#include +#include + +#include + +#define TS_STATS_SHMEM_MAGIC 0x54534F42 /* "TSOB" — initialized-segment latch */ + +#define TS_STATS_BUCKETS 8 +#define TS_STATS_PROBE_WIDTH 8 +#define TS_STATS_CACHE_LINE_SIZE 64 +#define TS_STATS_INVALID_IDX (-1) + +/* note that the code expects TS_STATS_BUCKETS and TS_STATS_PROBE_WIDTH + * to be small, < 256 numbers and uses uint8 at multiple places to store them. + */ +StaticAssertDecl(TS_STATS_BUCKETS < 256 && TS_STATS_PROBE_WIDTH < 256, + "TS_STATS_BUCKETS and TS_STATS_PROBE_WIDTH must be less than 256"); + +/* + * Chunk information is stored in a shared-memory segment, one per database. + * + * The information is organized into two parallel arrays (they logically + * belong together), one stores metadata about each chunk such as its ID and + * a sequence number, and the other stores the actual statistics. The actual + * stats are being read/written after the lookup on the metadata is done, so + * the metadata is held together for better cache locality. + * + * The table is organised into TS_STATS_BUCKETS buckets, and we keep track of + * the last update sequence number for these buckets. In each bucket the chunk id + * (compressed_relid) is hashed to find the 'home location' within the bucket. + * If the bucket is occupied by another chunk, we do a linear probe of up to + * TS_STATS_PROBE_WIDTH slots within the bucket. If the chunk is not found + * we proceed to the next bucket. We traverse the buckets in recency order. + * If we still can't find the chunk after traversing all buckets, it doesn't + * exist. On upsert, we check the entries in the last (the oldest bucket) + * within the search window (TS_STATS_PROBE_WIDTH slots) for eviction + * candidates. If we find an empty slot, we use it. If not we evict the oldest + * entry in the search window of the last bucket. + * + * Once we updated an entry in a bucket we update the last update sequence + * number for the bucket, so the next upsert will traverse the buckets in + * the order of recency. + */ + +typedef struct TsStatsRelids +{ + Oid compressed_relid; + Oid uncompressed_relid; +} TsStatsRelids; + +/* + * As explained above, TsStatsChunkMetadata is split off from TsStatsChunk + * because these are needed more frequently than the rest of the structure, + * because during traversal we read these fields first to find the entry. + * Keeping them together allows better cache locality for the common case. + * + * The 'state' field is composed of two parts: the lower two bits are flags, + * and the rest is a version counter. The valid flag is set when the slot is + * initialized with valid data, and being cleared at eviction. The version + * counter is incremented at every update, and readers can use it to order + * updates. + */ +typedef struct TsStatsChunkMetadata +{ + /* state layout: + * bit 0: is valid (valid when set) + * bit 1: in progress (being updated, not safe to read or evict) + * bits 2-63: seqno, incremented at every update for recency tracking + */ + pg_atomic_uint64 state; + /* Chunks are identified by relids.compressed_relid */ + TsStatsRelids relids; +} TsStatsChunkMetadata; + +/* Bit flag values as per the above: */ +#define TS_STATS_CHUNK_METADATA_VALID_FLAG ((uint64) 1) +#define TS_STATS_CHUNK_METADATA_IN_PROGRESS_FLAG ((uint64) 2) +/* Bit mask helpers */ +#define TS_STATS_CHUNK_METADATA_SEQNO_SHIFT 2 +#define TS_STATS_CHUNK_METADATA_SEQNO_MASK (~((1ULL << TS_STATS_CHUNK_METADATA_SEQNO_SHIFT) - 1)) +#define TS_STATS_CHUNK_METADATA_GET_SEQNO(state) \ + ((state & TS_STATS_CHUNK_METADATA_SEQNO_MASK) >> TS_STATS_CHUNK_METADATA_SEQNO_SHIFT) +#define TS_STATS_CHUNK_METADATA_IS_VALID(state) ((state & TS_STATS_CHUNK_METADATA_VALID_FLAG) != 0) +#define TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state) \ + ((state & TS_STATS_CHUNK_METADATA_IN_PROGRESS_FLAG) != 0) + +/* + * MinMaxSumDataWA uses atomics, but updates to this whole structure are not atomic. + * WA means 'With Atomics'. The potential inconsistency is acceptable for our use + * case, and it allows us to update the stats without acquiring an exclusive + * lock, which reduces contention and improves performance. + */ +typedef struct MinMaxSumDataWA +{ + /* + * The sqsum_val is a sum of squares, which is calculated as + * a double value and then cast to uint64 for atomic operations. + */ + pg_atomic_uint64 sqsum_val; + pg_atomic_uint64 sum_val; + pg_atomic_uint32 min_val; + pg_atomic_uint32 max_val; +} MinMaxSumDataWA; + +typedef struct MinMaxSumData +{ + double sqsum_val; + uint64 sum_val; + uint32 min_val; + uint32 max_val; +} MinMaxSumData; + +/* + * SharedCountersWA uses atomics, and mirrors the structure of SharedCounters. + * Updates to this structure are not atomic, WA means 'With Atomics'. The + * potential inconsistency is acceptable for our use case, and it allows us to + * update the stats without acquiring an exclusive lock. + */ +typedef struct SharedCountersWA +{ + pg_atomic_uint64 batches_deleted; + pg_atomic_uint64 batches_decompressed; + pg_atomic_uint64 tuples_decompressed; + pg_atomic_uint64 batches_scanned; + pg_atomic_uint64 batches_checked_by_bloom; + pg_atomic_uint64 batches_pruned_by_bloom; + pg_atomic_uint64 batches_without_bloom; + pg_atomic_uint64 batches_bloom_false_positives; + pg_atomic_uint64 batches_filtered_compressed; + pg_atomic_uint64 batches_filtered_decompressed; +} SharedCountersWA; + +typedef struct TsStatsChunk +{ + /* + * Microseconds since segment->base_timestamp. These are + * not used in CAS, only EQ and Assign, we can afford + * these to be non atomic. + */ + uint64 first_update_us; + uint64 last_update_us; + + /* Chunk stats at compression, totals: */ + pg_atomic_uint32 compressed_batch_count; + pg_atomic_uint32 compressed_block_count; + + /* byte stat across all batches and columns: */ + MinMaxSumDataWA compressed_block_bytes; + + /* sum bytes for each column, across batches: */ + MinMaxSumDataWA compressed_batch_bytes; + + /* rows across batches: */ + MinMaxSumDataWA compressed_batch_rows; + + /* Aggregated Cmd stats */ + SharedCountersWA cmd_totals; + + /* Cmd stats for the last operation */ + SharedCountersWA cmd_last_op; + + /* Operation counts */ + pg_atomic_uint32 n_selects; + pg_atomic_uint32 n_inserts; + pg_atomic_uint32 n_updates; + pg_atomic_uint32 n_deletes; + +} TsStatsChunk; + +typedef struct TsStatsChunkBucketMetadata +{ + pg_atomic_uint64 last_update_seqno; + /* padding to avoid false sharing on 64 byte cache lines */ + uint8 padding[56]; +} TsStatsChunkBucketMetadata; + +typedef struct TsStatsChunkSegment +{ + /* Magic set once at init, never written again*/ + uint32 magic; /* TS_STATS_SHMEM_MAGIC when initialized */ + Oid dboid; /* database that owns this segment */ + + /* Only used during whole segment operations, such as initialization. */ + LWLock lock; + + /* base timestamp for *_update_us fields */ + TimestampTz base_timestamp; + + /* Number of slots */ + uint32 num_slots; + + /* Sequence number shared across all chunk slots */ + pg_atomic_uint64 update_seqno; + + /* + * Trailing arrays — dynamically sized and aligned to cache lines. + * + * The logical layout is: + * + * TsStatsChunkMetadata meta[num_slots]; + * TsStatsChunk slots[num_slots]; + * TsStatsChunkBucketMetadata buckets[TS_STATS_BUCKETS]; + * + */ +} TsStatsChunkSegment; + +#define TS_STATS_CACHE_PADDED(S) (TYPEALIGN((TS_STATS_CACHE_LINE_SIZE), (S))) + +static inline Size +ts_stats_chunk_data_size(uint32 num_slots) +{ + Size size = 0; + size += TS_STATS_CACHE_PADDED(num_slots * sizeof(TsStatsChunkMetadata)); + size += TS_STATS_CACHE_PADDED(num_slots * sizeof(TsStatsChunk)); + size += TS_STATS_CACHE_PADDED(TS_STATS_BUCKETS * sizeof(TsStatsChunkBucketMetadata)); + return size; +} + +static inline Size +ts_stats_chunk_segment_size(uint32 num_slots) +{ + Size size = TS_STATS_CACHE_PADDED(sizeof(TsStatsChunkSegment)); + return add_size(size, ts_stats_chunk_data_size(num_slots)); +} + +static inline TsStatsChunkMetadata * +ts_stats_chunk_metadata(TsStatsChunkSegment *seg) +{ + return (TsStatsChunkMetadata *) ((char *) seg + + TS_STATS_CACHE_PADDED(sizeof(TsStatsChunkSegment))); +} + +static inline TsStatsChunk * +ts_stats_chunk_slots(TsStatsChunkSegment *seg) +{ + return (TsStatsChunk *) ((char *) seg + TS_STATS_CACHE_PADDED(sizeof(TsStatsChunkSegment)) + + TS_STATS_CACHE_PADDED(seg->num_slots * sizeof(TsStatsChunkMetadata))); +} + +static inline TsStatsChunkBucketMetadata * +ts_stats_chunk_bucket_metadata(TsStatsChunkSegment *seg) +{ + return (TsStatsChunkBucketMetadata *) ((char *) seg + + TS_STATS_CACHE_PADDED(sizeof(TsStatsChunkSegment)) + + TS_STATS_CACHE_PADDED(seg->num_slots * + sizeof(TsStatsChunkMetadata)) + + TS_STATS_CACHE_PADDED(seg->num_slots * + sizeof(TsStatsChunk))); +} + +#undef TS_STATS_CACHE_PADDED + +static inline void +ts_stats_chunk_init_slot(TsStatsChunk *slot) +{ + /* memset all to zero, except the min values */ + memset(slot, 0, sizeof(TsStatsChunk)); + pg_atomic_write_u32(&slot->compressed_block_bytes.min_val, UINT32_MAX); + pg_atomic_write_u32(&slot->compressed_batch_bytes.min_val, UINT32_MAX); + pg_atomic_write_u32(&slot->compressed_batch_rows.min_val, UINT32_MAX); +} + +static inline void +init_min_max_sum(MinMaxSumData *mmsd) +{ + mmsd->sqsum_val = 0; + mmsd->sum_val = 0; + mmsd->min_val = UINT32_MAX; + mmsd->max_val = 0; +} + +static inline void +mms_observe(MinMaxSumData *m, uint32 value) +{ + if (value < m->min_val) + { + m->min_val = value; + } + if (value > m->max_val) + { + m->max_val = value; + } + m->sum_val += value; + + double square_value = (double) value * (double) value; + m->sqsum_val += square_value; +} + +#define TS_STATS_MMS_CAS_MAX_RETRIES 5 + +static inline void +mms_merge(MinMaxSumDataWA *dst, const MinMaxSumData *src) +{ + /* + * this function does a bounded CAS loop so it won't spin indefinitely and does a best + * effort update of the min, max, sum, and sqsum values. + */ + uint32 cur_min = pg_atomic_read_u32(&dst->min_val); + for (int r = 0; src->min_val < cur_min && r < TS_STATS_MMS_CAS_MAX_RETRIES; r++) + { + if (pg_atomic_compare_exchange_u32(&dst->min_val, &cur_min, src->min_val)) + { + break; + } + /* CAS failed, stay with the old value */ + } + + uint32 cur_max = pg_atomic_read_u32(&dst->max_val); + for (int r = 0; src->max_val > cur_max && r < TS_STATS_MMS_CAS_MAX_RETRIES; r++) + { + if (pg_atomic_compare_exchange_u32(&dst->max_val, &cur_max, src->max_val)) + { + break; + } + /* CAS failed, stay with the old value */ + } + + pg_atomic_add_fetch_u64(&dst->sum_val, src->sum_val); + + uint64 cur_sqsum = pg_atomic_read_u64(&dst->sqsum_val); + for (int r = 0; r < TS_STATS_MMS_CAS_MAX_RETRIES; r++) + { + double new_d; + memcpy(&new_d, &cur_sqsum, sizeof(double)); + new_d += src->sqsum_val; + + uint64 desired; + memcpy(&desired, &new_d, sizeof(uint64)); + + if (pg_atomic_compare_exchange_u64(&dst->sqsum_val, &cur_sqsum, desired)) + { + break; + } + /* CAS failed, stay with the old value */ + } +} + +#undef TS_STATS_MMS_CAS_MAX_RETRIES + +static inline void +mms_update(MinMaxSumDataWA *dst, const MinMaxSumData *src) +{ + pg_atomic_write_u32(&dst->min_val, src->min_val); + pg_atomic_write_u32(&dst->max_val, src->max_val); + pg_atomic_write_u64(&dst->sum_val, src->sum_val); + + /* sqsum is calculated as a double and then copied to uint64 for atomic operations */ + pg_atomic_write_u64(&dst->sqsum_val, *((uint64 *) &src->sqsum_val)); +} + +static inline double +mms_sqsum(MinMaxSumDataWA *m) +{ + uint64 sqsum_val = pg_atomic_read_u64(&m->sqsum_val); + return *((double *) &sqsum_val); +} diff --git a/src/ts_stats/ts_stats_record.c b/src/ts_stats/ts_stats_record.c new file mode 100644 index 00000000000..930f5e48efb --- /dev/null +++ b/src/ts_stats/ts_stats_record.c @@ -0,0 +1,338 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#include "ts_stats_record.h" +#include "ts_stats/ts_stats_defs.h" +#include "ts_stats_segment.h" + +#include +#include +#include +#include +#include + +static inline uint64 +now_us_offset(TsStatsChunkSegment *seg) +{ + return (uint64) (GetCurrentTimestamp() - seg->base_timestamp); +} + +static inline void +slot_touch_timestamps(TsStatsChunk *slot, uint64 now_us) +{ + /* There is very slight chance that this function is being called, just in the + * same microsecond as the base timestamp. */ + if (now_us == 0) + { + now_us = 1; + } + + if (slot->first_update_us == 0) + { + slot->first_update_us = now_us; + } + slot->last_update_us = now_us; +} + +void +ts_stats_chunk_record_compression(TsStatsRelids relids, const CompressionStatsAccumulator *a) +{ + if (!OidIsValid(relids.compressed_relid) || !OidIsValid(relids.uncompressed_relid) || + a->batch_count == 0) + { + return; + } + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return; + } + + /* get the slot_index for the upsert. this sets the in-progress flag and updates the bucket's + * metadata if successful. + */ + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + if (slot_idx == TS_STATS_INVALID_IDX) + { + /* if we failed to find or insert a slot for this compressed_relid, we just give up this + * time */ + return; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + /* get the timestamp outside the lock scope */ + uint64 now_us = now_us_offset(seg); + + /* get the base pointers for the chunk */ + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + + /* since we have a valid slot_idx, it means the admin stuff is done with the slot, and we only + * need to update the actual statistics. the in-progress flag is set for the slot which we + * will need to clear once the update is done. + */ + TsStatsChunk *slot = &slot_base[slot_idx]; + pg_atomic_add_fetch_u32(&slot->compressed_batch_count, a->batch_count); + pg_atomic_add_fetch_u32(&slot->compressed_block_count, a->block_count); + + /* Fold the three accumulator triples into the matching slot triples. */ + mms_merge(&slot->compressed_block_bytes, &a->block_size); + mms_merge(&slot->compressed_batch_bytes, &a->batch_size); + mms_merge(&slot->compressed_batch_rows, &a->batch_rows); + + slot_touch_timestamps(slot, now_us); + + /* clear the in-progress flag */ + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); +} + +void +ts_stats_chunk_record_cmd(TsStatsRelids relids, CmdType cmd, const SharedCounters *c) +{ + if (!OidIsValid(relids.compressed_relid) || !OidIsValid(relids.uncompressed_relid) || c == NULL) + { + return; + } + if (cmd != CMD_SELECT && cmd != CMD_INSERT && cmd != CMD_UPDATE && cmd != CMD_DELETE) + { + return; + } + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return; + } + + /* get the slot_index for the upsert. this sets the in-progress flag and updates the bucket's + * metadata if successful. + */ + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + if (slot_idx == TS_STATS_INVALID_IDX) + { + /* if we failed to find or insert a slot for this compressed_relid, we just give up this + * time */ + return; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + /* get the timestamp outside the lock scope */ + uint64 now_us = now_us_offset(seg); + + /* get the base pointers for the chunk */ + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + + /* since we have a valid slot_idx, it means the admin stuff is done with the slot, and we only + * need to update the actual statistics. the in-progress flag is set for the slot which we + * will need to clear once the update is done. + */ + TsStatsChunk *slot = &slot_base[slot_idx]; + + switch (cmd) + { + case CMD_SELECT: + pg_atomic_add_fetch_u32(&slot->n_selects, 1); + break; + case CMD_INSERT: + pg_atomic_add_fetch_u32(&slot->n_inserts, 1); + break; + case CMD_UPDATE: + pg_atomic_add_fetch_u32(&slot->n_updates, 1); + break; + case CMD_DELETE: + pg_atomic_add_fetch_u32(&slot->n_deletes, 1); + break; + default: + break; + } + + /* update totals */ + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_deleted, c->batches_deleted); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_decompressed, c->batches_decompressed); + pg_atomic_add_fetch_u64(&slot->cmd_totals.tuples_decompressed, c->tuples_decompressed); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_scanned, c->batches_scanned); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_checked_by_bloom, + c->batches_checked_by_bloom); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_pruned_by_bloom, c->batches_pruned_by_bloom); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_without_bloom, c->batches_without_bloom); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_bloom_false_positives, + c->batches_bloom_false_positives); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_filtered_compressed, + c->batches_filtered_compressed); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_filtered_decompressed, + c->batches_filtered_decompressed); + + /* update last op */ + pg_atomic_write_u64(&slot->cmd_last_op.batches_deleted, c->batches_deleted); + pg_atomic_write_u64(&slot->cmd_last_op.batches_decompressed, c->batches_decompressed); + pg_atomic_write_u64(&slot->cmd_last_op.tuples_decompressed, c->tuples_decompressed); + pg_atomic_write_u64(&slot->cmd_last_op.batches_scanned, c->batches_scanned); + pg_atomic_write_u64(&slot->cmd_last_op.batches_checked_by_bloom, c->batches_checked_by_bloom); + pg_atomic_write_u64(&slot->cmd_last_op.batches_pruned_by_bloom, c->batches_pruned_by_bloom); + pg_atomic_write_u64(&slot->cmd_last_op.batches_without_bloom, c->batches_without_bloom); + pg_atomic_write_u64(&slot->cmd_last_op.batches_bloom_false_positives, + c->batches_bloom_false_positives); + pg_atomic_write_u64(&slot->cmd_last_op.batches_filtered_compressed, + c->batches_filtered_compressed); + pg_atomic_write_u64(&slot->cmd_last_op.batches_filtered_decompressed, + c->batches_filtered_decompressed); + + /* update timestamps */ + slot_touch_timestamps(slot, now_us); + + /* clear the in-progress flag */ + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); +} + +void +ts_stats_chunk_record_decompression(TsStatsRelids relids, const CompressionStatsAccumulator *a) +{ + if (!OidIsValid(relids.compressed_relid) || !OidIsValid(relids.uncompressed_relid)) + { + return; + } + if (a->batch_count == 0 && a->block_count == 0) + { + return; + } + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return; + } + + /* get the slot_index for the upsert. this sets the in-progress flag and updates the bucket's + * metadata if successful. + */ + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + if (slot_idx == TS_STATS_INVALID_IDX) + { + /* if we failed to find or insert a slot for this compressed_relid, we just give up this + * time */ + return; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + /* get the timestamp outside the lock scope */ + uint64 now_us = now_us_offset(seg); + + /* get the base pointers for the chunk */ + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + + /* since we have a valid slot_idx, it means the admin stuff is done with the slot, and we only + * need to update the actual statistics. the in-progress flag is set for the slot which we + * will need to clear once the update is done. + */ + TsStatsChunk *slot = &slot_base[slot_idx]; + + /* Batch group being updated if the current information is more complete. */ + if (a->batch_count > pg_atomic_read_u32(&slot->compressed_batch_count) && + a->batch_rows.sum_val > pg_atomic_read_u64(&slot->compressed_batch_rows.sum_val) && + a->batch_rows.sqsum_val > mms_sqsum(&slot->compressed_batch_rows) && + a->batch_size.sum_val > pg_atomic_read_u64(&slot->compressed_batch_bytes.sum_val) && + a->batch_size.sqsum_val > mms_sqsum(&slot->compressed_batch_bytes)) + { + pg_atomic_write_u32(&slot->compressed_batch_count, a->batch_count); + mms_update(&slot->compressed_batch_rows, &a->batch_rows); + mms_update(&slot->compressed_batch_bytes, &a->batch_size); + } + + /* Column group being updated if the current information is more complete. */ + if (a->block_count > pg_atomic_read_u32(&slot->compressed_block_count) && + a->block_size.sum_val > pg_atomic_read_u64(&slot->compressed_block_bytes.sum_val) && + a->block_size.sqsum_val > mms_sqsum(&slot->compressed_block_bytes)) + { + pg_atomic_write_u32(&slot->compressed_block_count, a->block_count); + mms_update(&slot->compressed_block_bytes, &a->block_size); + } + + /* update timestamps */ + slot_touch_timestamps(slot, now_us); + + /* clear the in-progress flag */ + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); +} + +bool +ts_stats_chunk_lookup(Oid compressed_relid, TsStatsChunk *out) +{ + if (!OidIsValid(compressed_relid) || out == NULL) + { + return false; + } + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return false; + } + + int32 slot_idx = ts_stats_chunk_segment_lookup(seg, compressed_relid); + + if (slot_idx == TS_STATS_INVALID_IDX) + { + return false; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + /* get the base pointers for the chunk */ + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + TsStatsChunk *slot = &slot_base[slot_idx]; + + /* dumb memcpy, hoping the data is still consistent */ + memcpy(out, slot, sizeof(TsStatsChunk)); + return true; +} + +void +ts_stats_chunk_evict(Oid compressed_relid) +{ + if (!OidIsValid(compressed_relid)) + { + return; + } + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return; + } + + int32 slot_idx = ts_stats_chunk_segment_lookup(seg, compressed_relid); + + if (slot_idx == TS_STATS_INVALID_IDX) + { + return; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + TsStatsChunkMetadata *meta = &meta_base[slot_idx]; + + /* limited CAS loop to ensure we don't evict a chunk that is currently being updated, the danger + * is that the one that we are evicting is already being overwritten by a newer/other chunk. + */ + uint64 state = pg_atomic_read_u64(&meta->state); + uint64 new_seqno = pg_atomic_add_fetch_u64(&seg->update_seqno, 1); + uint64 new_state = (new_seqno << TS_STATS_CHUNK_METADATA_SEQNO_SHIFT); + for (int i = 0; + i < 5 && + /* the slot still matches the compressed_relid */ + meta->relids.compressed_relid == compressed_relid && + /* the slot is still valid and not in progress */ + TS_STATS_CHUNK_METADATA_IS_VALID(state) && + !(TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) && + /* the slot being updated is older than the sequence number of the current update */ + TS_STATS_CHUNK_METADATA_GET_SEQNO(state) < new_seqno; + ++i) + { + if (pg_atomic_compare_exchange_u64(&meta->state, &state, new_state)) + { + meta->relids.compressed_relid = InvalidOid; + break; + } + } +} diff --git a/src/ts_stats/ts_stats_record.h b/src/ts_stats/ts_stats_record.h new file mode 100644 index 00000000000..1239d8a8e1e --- /dev/null +++ b/src/ts_stats/ts_stats_record.h @@ -0,0 +1,78 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ +#pragma once + +#include "ts_stats_defs.h" +#include /* CmdType */ + +typedef struct CompressionStatsAccumulator +{ + /* Per-column: updated on every column write. */ + uint32 block_count; + MinMaxSumData block_size; + + /* Per-compressed-row: updated on every batch flush. */ + uint32 batch_count; + MinMaxSumData batch_size; + MinMaxSumData batch_rows; + + /* Reused between the per-column and per-comp-row. */ + uint32 pending_batch_bytes; + +} CompressionStatsAccumulator; + +static inline void +ts_stats_compression_acc_init(CompressionStatsAccumulator *a) +{ + memset(a, 0, sizeof(*a)); + init_min_max_sum(&a->block_size); + init_min_max_sum(&a->batch_size); + init_min_max_sum(&a->batch_rows); +} + +static inline void +ts_stats_compression_acc_column(CompressionStatsAccumulator *a, uint32 block_bytes) +{ + a->block_count++; + mms_observe(&a->block_size, block_bytes); + a->pending_batch_bytes += block_bytes; +} + +static inline void +ts_stats_compression_acc_batch(CompressionStatsAccumulator *a, uint32 batch_rows) +{ + a->batch_count++; + mms_observe(&a->batch_size, a->pending_batch_bytes); + mms_observe(&a->batch_rows, batch_rows); + a->pending_batch_bytes = 0; +} + +/* Update shared memory with new statistics information. */ + +/* Compression — commits the accumulator into the chunk's slot. */ +extern TSDLLEXPORT void ts_stats_chunk_record_compression(TsStatsRelids relids, + const CompressionStatsAccumulator *a); + +/* Cmd stats on compressed data. cmd is one of CMD_INSERT / CMD_UPDATE / CMD_DELETE / CMD_SELECT */ +extern TSDLLEXPORT void ts_stats_chunk_record_cmd(TsStatsRelids relids, CmdType cmd, + const SharedCounters *c); + +/* For backfilling compressed data during decompression. + * This function will update the compression stats for the chunk when it finds more complete + * information about the compressed data than the one we have in memory. The reason we + * need this is because the in-memory entries may miss the compression event, either because + * of a restart or because the chunk information has been evicted from memory due to lack of use. + */ +extern TSDLLEXPORT void ts_stats_chunk_record_decompression(TsStatsRelids relids, + const CompressionStatsAccumulator *a); + +/* Lookup TsStatsChunk based on the compressed relid. It returns true if the chunk is found, false + * otherwise. */ +extern TSDLLEXPORT bool ts_stats_chunk_lookup(Oid compressed_relid, TsStatsChunk *out); + +/* Evict the chunk information for a given compressed relid. This function is called when we + * drop a compressed chunk. */ +extern TSDLLEXPORT void ts_stats_chunk_evict(Oid compressed_relid); diff --git a/src/ts_stats/ts_stats_segment.c b/src/ts_stats/ts_stats_segment.c new file mode 100644 index 00000000000..338d85bece2 --- /dev/null +++ b/src/ts_stats/ts_stats_segment.c @@ -0,0 +1,685 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#include "ts_stats_segment.h" +#include "guc.h" /* ts_guc_stats_max_chunks */ + +#include +#include "utils/syscache.h" +#include +#include +#include + +#if PG17_GE +#include +#else +#include "../loader/ts_stats_handles.h" +#include +#include +#endif + +/* + * One-time init. + */ +static void +ts_stats_chunk_init_segment(void *ptr) +{ + TsStatsChunkSegment *seg = (TsStatsChunkSegment *) ptr; + uint32 num_slots = (uint32) ts_guc_stats_max_chunks; + + /* must be powers of two */ + Assert((num_slots & (num_slots - 1)) == 0); + + /* zeros are valid for all variables, in particular this clears + * the validity flag part of the TsStatsChunkMetadata.state field */ + memset(seg, 0, ts_stats_chunk_segment_size(num_slots)); + + seg->magic = TS_STATS_SHMEM_MAGIC; + seg->dboid = MyDatabaseId; + LWLockInitialize(&seg->lock, LWLockNewTrancheId()); + LWLockRegisterTranche(seg->lock.tranche, "ts_stats_chunk"); + seg->base_timestamp = GetCurrentTimestamp(); + seg->num_slots = num_slots; + + /* sequence number starts at 1 */ + pg_atomic_write_u64(&seg->update_seqno, 1); +} + +void +ts_stats_chunk_segment_reset(TsStatsChunkSegment *seg) +{ + /* blindly overwrite all dynamic state, no locking and no checks */ + void *meta_base = ts_stats_chunk_metadata(seg); + Size data_size = ts_stats_chunk_data_size(seg->num_slots); + memset(meta_base, 0, data_size); +} + +/* + * Backend-local cache. MyDatabaseId is fixed for a backend's lifetime, so + * cached_segment is safe to reuse across calls. + */ +static TsStatsChunkSegment *cached_segment = NULL; + +/* GetNamedDSMSegment was introduced in PG 17, so we need separate + * logic for older versions. */ +#if PG17_GE + +TsStatsChunkSegment * +ts_get_stats_chunk_segment(void) +{ + if (!IS_STATS_CHUNKS_ENABLED()) + { + return NULL; + } + + if (likely(cached_segment != NULL)) + { + return cached_segment; + } + + Size seg_size = ts_stats_chunk_segment_size((uint32) ts_guc_stats_max_chunks); + + char name[64]; + snprintf(name, sizeof(name), "ts_stats_chunk_db_%u", MyDatabaseId); + + bool found; + cached_segment = GetNamedDSMSegment(name, seg_size, ts_stats_chunk_init_segment, &found); + + if (cached_segment != NULL && cached_segment->magic != TS_STATS_SHMEM_MAGIC) + { + elog(WARNING, + "ts_stats_chunk: segment for database %u has unexpected magic, disabling", + MyDatabaseId); + cached_segment = NULL; + } + return cached_segment; +} + +#else /* PG17_LT */ + +/* The below section is about handling older PostgreSQL version, before PG 17. + * The memory segments for each database are registered in a global handle table. + * When the handle table is full, we sweep through the entries and remove the + * segments for the databases that have been dropped. If there are more than + * TS_STATS_MAX_DATABASES databases, or if the segment creation fails for any + * reason, we disable stats tracking for the current database. + */ + +static TsObservHandleEntry * +ts_stats_chunk_find_free_entry(TsObservHandleTable *tbl) +{ + for (uint32 i = 0; i < tbl->max_entries; i++) + { + if (!OidIsValid(tbl->entries[i].dboid)) + { + return &tbl->entries[i]; + } + } + return NULL; +} + +static TsObservHandleEntry * +ts_stats_chunk_find_entry(TsObservHandleTable *tbl, Oid dboid, int segment_type) +{ + for (uint32 i = 0; i < tbl->max_entries; i++) + { + if (tbl->entries[i].dboid == dboid && tbl->entries[i].segment_type == segment_type) + { + return &tbl->entries[i]; + } + } + return NULL; +} + +static void +ts_stats_chunk_sweep_stale(TsObservHandleTable *tbl) +{ + Assert(IsTransactionState()); + + typedef struct + { + uint32 idx; + Oid dboid; + } SnapEntry; + + SnapEntry snap[TS_STATS_MAX_DATABASES]; + SnapEntry gone[TS_STATS_MAX_DATABASES]; + dsm_handle unpin[TS_STATS_MAX_DATABASES]; + uint32 snap_n = 0, gone_n = 0, unpin_n = 0; + + /* Sweep through the handle table and take a snapshot of valid entries */ + LWLockAcquire(&tbl->lock, LW_EXCLUSIVE); + for (uint32 i = 0; i < TS_STATS_MAX_DATABASES; i++) + { + if (OidIsValid(tbl->entries[i].dboid)) + { + snap[snap_n++] = (SnapEntry){ i, tbl->entries[i].dboid }; + } + } + LWLockRelease(&tbl->lock); + + /* Check which entries are no longer valid */ + for (uint32 j = 0; j < snap_n; j++) + { + HeapTuple tup = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(snap[j].dboid)); + if (!HeapTupleIsValid(tup)) + { + gone[gone_n++] = snap[j]; + } + else + { + ReleaseSysCache(tup); + } + } + + /* Remove stale entries from the handle table */ + LWLockAcquire(&tbl->lock, LW_EXCLUSIVE); + for (uint32 k = 0; k < gone_n; k++) + { + TsObservHandleEntry *e = &tbl->entries[gone[k].idx]; + if (e->dboid == gone[k].dboid) + { + unpin[unpin_n++] = e->handle; + e->dboid = InvalidOid; + e->handle = 0; + } + } + LWLockRelease(&tbl->lock); + + /* Unpin the stale segments */ + for (uint32 m = 0; m < unpin_n; m++) + { + dsm_unpin_segment(unpin[m]); + } +} + +static TsStatsChunkSegment * +ts_stats_chunk_create_segment(TsObservHandleTable *tbl) +{ + Size seg_size = ts_stats_chunk_segment_size((uint32) ts_guc_stats_max_chunks); + + dsm_segment *local_seg = dsm_create(seg_size, 0); + if (local_seg == NULL) + { + return NULL; + } + + dsm_pin_segment(local_seg); + dsm_pin_mapping(local_seg); + dsm_handle local_handle = dsm_segment_handle(local_seg); + + TsStatsChunkSegment *obs = dsm_segment_address(local_seg); + ts_stats_chunk_init_segment(obs); + + LWLockAcquire(&tbl->lock, LW_EXCLUSIVE); + + TsObservHandleEntry *entry = + ts_stats_chunk_find_entry(tbl, MyDatabaseId, TS_STATS_TYPE_CHUNK_SEGMENT); + if (entry != NULL) + { + dsm_handle winner_handle = entry->handle; + LWLockRelease(&tbl->lock); + + dsm_unpin_segment(local_handle); + dsm_detach(local_seg); + + dsm_segment *winner_seg = dsm_attach(winner_handle); + if (winner_seg == NULL) + { + return NULL; + } + dsm_pin_mapping(winner_seg); + return dsm_segment_address(winner_seg); + } + + /* Find a free slot, with one sweep retry. */ + entry = ts_stats_chunk_find_free_entry(tbl); + if (entry == NULL) + { + LWLockRelease(&tbl->lock); + ts_stats_chunk_sweep_stale(tbl); + LWLockAcquire(&tbl->lock, LW_EXCLUSIVE); + entry = ts_stats_chunk_find_free_entry(tbl); + } + + /* Table full. */ + if (entry == NULL) + { + LWLockRelease(&tbl->lock); + dsm_unpin_segment(local_handle); + dsm_detach(local_seg); + elog(LOG, + "ts_stats_chunk: handle table full after sweep, chunk statistics disabled " + "for database %u", + MyDatabaseId); + return NULL; + } + + /* Publish. */ + entry->dboid = MyDatabaseId; + entry->handle = local_handle; + entry->segment_type = TS_STATS_TYPE_CHUNK_SEGMENT; + LWLockRelease(&tbl->lock); + return obs; +} + +TsStatsChunkSegment * +ts_get_stats_chunk_segment(void) +{ + static bool rendezvous_checked = false; + + if (!IS_STATS_CHUNKS_ENABLED()) + { + return NULL; + } + + if (likely(cached_segment != NULL)) + { + return cached_segment; + } + + if (rendezvous_checked) + { + return NULL; + } + rendezvous_checked = true; + + TsObservHandleTable **rv = + (TsObservHandleTable **) find_rendezvous_variable("ts_stats_handles"); + TsObservHandleTable *tbl = *rv; + if (tbl == NULL) + { + return NULL; + } + + LWLockAcquire(&tbl->lock, LW_SHARED); + TsObservHandleEntry *entry = + ts_stats_chunk_find_entry(tbl, MyDatabaseId, TS_STATS_TYPE_CHUNK_SEGMENT); + + if (entry != NULL) + { + dsm_segment *seg = dsm_find_mapping(entry->handle); + if (seg == NULL) + { + seg = dsm_attach(entry->handle); + if (seg != NULL) + { + dsm_pin_mapping(seg); + } + } + LWLockRelease(&tbl->lock); + + if (seg != NULL) + { + cached_segment = dsm_segment_address(seg); + if (cached_segment->magic != TS_STATS_SHMEM_MAGIC) + { + elog(WARNING, + "ts_stats_chunk: segment for database %u has unexpected magic, disabling", + MyDatabaseId); + cached_segment = NULL; + } + return cached_segment; + } + } + else + { + LWLockRelease(&tbl->lock); + } + + cached_segment = ts_stats_chunk_create_segment(tbl); + return cached_segment; +} + +#endif /* PG17_GE */ + +/* + * Knuth's multiplicative hash. The constant 0x9E3779B9 = floor(2^32 / phi). + * Multiplying scrambles input structure across the 32-bit space. We don't + * need perfect hashing here, just a good enough distribution that puts + * consecutive relids at least window width apart to avoid clustering in + * the linear probing. + */ +static inline uint32 +hash_relid(const TsStatsChunkSegment *seg, Oid compressed_relid) +{ + uint32 bucket_size = seg->num_slots / TS_STATS_BUCKETS; + uint32 product = (uint32) compressed_relid * 0x9E3779B9U; + return (uint32) (((uint64_t) product * (uint64_t) bucket_size) >> 32); +} + +/* Return the sorted bucket order as an array of uint8 indices */ +static inline void +get_sorted_bucket_in_desc_order(TsStatsChunkSegment *seg, uint8 *out) +{ + Assert(out != NULL && seg != NULL); + uint64 seqnos[TS_STATS_BUCKETS]; + TsStatsChunkBucketMetadata *bucket_base = ts_stats_chunk_bucket_metadata(seg); + for (int i = 0; i < TS_STATS_BUCKETS; i++) + { + out[i] = (uint8) i; + seqnos[i] = pg_atomic_read_u64(&bucket_base[i].last_update_seqno); + } + + /* sort the bucket index by the last update sequence number in descending order */ + for (int j = 0; j < TS_STATS_BUCKETS - 1; j++) + { + for (int k = 0; k < TS_STATS_BUCKETS - j - 1; k++) + { + if (seqnos[out[k]] < seqnos[out[k + 1]]) + { + uint8 temp = out[k]; + out[k] = out[k + 1]; + out[k + 1] = temp; + } + } + } +} + +/* This function evicts/updates a chunk at the specified index and also sets the + * in-progress flag, plus updated the bucket's last update sequence number. + * This is a utility function to simplify 'ts_stats_chunk_segment_prepare_upsert' + */ +static inline int32 +ts_stats_chunk_update_at(TsStatsChunkSegment *seg, TsStatsRelids relids, int32 idx, + uint64 old_state) +{ + Assert(seg != NULL || OidIsValid(relids.compressed_relid) || + OidIsValid(relids.uncompressed_relid)); + if (seg == NULL || !OidIsValid(relids.compressed_relid) || + !OidIsValid(relids.uncompressed_relid)) + { + return TS_STATS_INVALID_IDX; + } + + uint32 bucket_size = seg->num_slots / TS_STATS_BUCKETS; + uint32 bucket_idx = idx / bucket_size; + uint64 new_seqno = pg_atomic_add_fetch_u64(&seg->update_seqno, 1); + uint64 new_state = (new_seqno << TS_STATS_CHUNK_METADATA_SEQNO_SHIFT) | + TS_STATS_CHUNK_METADATA_VALID_FLAG | + TS_STATS_CHUNK_METADATA_IN_PROGRESS_FLAG; + + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + TsStatsChunkBucketMetadata *bucket_base = ts_stats_chunk_bucket_metadata(seg); + + TsStatsChunkMetadata *meta = &meta_base[idx]; + + /* only evict when the state is the same as the old state, which means + * no other process has updated this slot since we read it last time */ + if (!pg_atomic_compare_exchange_u64(&meta->state, &old_state, new_state)) + { + return TS_STATS_INVALID_IDX; + } + + /* update the metadata */ + meta->relids = relids; + pg_atomic_write_u64(&bucket_base[bucket_idx].last_update_seqno, new_seqno); + + return (int32) idx; +} + +/* Public API */ +int32 +ts_stats_chunk_segment_prepare_upsert(TsStatsChunkSegment *seg, TsStatsRelids relids) +{ + Assert(seg != NULL); + if (seg == NULL || !OidIsValid(relids.compressed_relid) || + !OidIsValid(relids.uncompressed_relid)) + { + return TS_STATS_INVALID_IDX; + } + + /* get the base pointers */ + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + + /* sort the buckets by the last update sequence number in descending order */ + uint8 bucket_order[TS_STATS_BUCKETS]; + get_sorted_bucket_in_desc_order(seg, bucket_order); + + /* get the home location within the buckets */ + uint32 home = hash_relid(seg, relids.compressed_relid); + uint32 bucket_size = seg->num_slots / TS_STATS_BUCKETS; + + /* keep the empty positions as we scan forward. these will + * be tried if the chunk was not found. */ + uint32 empty_positions[TS_STATS_PROBE_WIDTH * TS_STATS_BUCKETS]; + uint32 empty_count = 0; + + /* iterate over the buckets in most-recently-updated order */ + for (uint8 i = 0; i < TS_STATS_BUCKETS; i++) + { + uint8 bucket_idx = bucket_order[i]; + uint32 start = bucket_idx * bucket_size; + + /* linear probe within the bucket */ + for (uint8 j = 0; j < TS_STATS_PROBE_WIDTH; j++) + { + uint32 idx = start + ((home + j) % bucket_size); + TsStatsChunkMetadata *meta = &meta_base[idx]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* record the empty position for later use, but we still need + * to continue scanning the remaining slots */ + empty_positions[empty_count++] = idx; + continue; + } + if (meta->relids.compressed_relid == relids.compressed_relid) + { + /* update the chunk metadata state to in-progress and the bucket's + * last update sequence number. + */ + if (ts_stats_chunk_update_at(seg, relids, idx, state) != TS_STATS_INVALID_IDX) + { + return (int32) idx; + } + + /* if the update failed, then another process has updated this slot. + * if the slot still belongs to the same compressed_relid, we can try + * to update it again, so the update sequence numbers get updated. + */ + state = pg_atomic_read_u64(&meta->state); + if (TS_STATS_CHUNK_METADATA_IS_VALID(state) && + meta->relids.compressed_relid == relids.compressed_relid) + { + if (ts_stats_chunk_update_at(seg, relids, idx, state) != TS_STATS_INVALID_IDX) + { + return (int32) idx; + } + } + + /* if this failed at the second time, we better leave it alone and proceed + * with the next slot. + */ + } + } + } + + /* at this point we haven't found the chunk so we first check the empty slots. + * they may be taken conncurrently by another process, so they need to be checked + * again. the order of the empty slots are somwewhat ordered as we gathered them + * in the bucket recency order. best to traverse them in this recency order so + * the buckets keep the recency property as much as possible. + */ + for (uint32 i = 0; i < empty_count; i++) + { + uint32 idx = empty_positions[i]; + TsStatsChunkMetadata *meta = &meta_base[idx]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* this slot is no longer empty, skip it */ + continue; + } + + /* try to claim this slot, and it will update its state to in-progress + * and the bucket's last update sequence number. + */ + if (ts_stats_chunk_update_at(seg, relids, idx, state) != TS_STATS_INVALID_IDX) + { + TsStatsChunk *slot = &slot_base[idx]; + ts_stats_chunk_init_slot(slot); + return (int32) idx; + } + } + + /* at this point we have not found the slot and neither empty ones to use + * we will need to evict (overwrite) one. the eviction only checks the + * oldest bucket's window and uses the slot that is the oldest and not + * in-progress. if every slot is in progress we try the next bucket in + * recency order. if none has an evictable slot, we give up and return + * invalid index. + * + * the worst case is that we checked the windows of all buckets and we + * tried to evict from all slots, so that is + * + * 2 x TS_STATS_BUCKETS x TS_STATS_PROBE_WIDTH checks, which is + * + * still acceptable, but very unlikely to happen. + */ + + uint8 oldest_bucket_idx = bucket_order[TS_STATS_BUCKETS - 1]; + uint32 start = oldest_bucket_idx * bucket_size; + int32 candidate_idx = TS_STATS_INVALID_IDX; + uint64 candidate_seqno = UINT64_MAX; + + for (uint8 j = 0; j < TS_STATS_PROBE_WIDTH; j++) + { + uint32 idx = start + ((home + j) % bucket_size); + TsStatsChunkMetadata *meta = &meta_base[idx]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* this slot is empty, although at this point it should not happen + * as the previous loops should have handled empty slots. */ + if (ts_stats_chunk_update_at(seg, relids, idx, state) != TS_STATS_INVALID_IDX) + { + TsStatsChunk *slot = &slot_base[idx]; + ts_stats_chunk_init_slot(slot); + return (int32) idx; + } + } + if (!TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* this slot is not in progress, mark as a candidate */ + uint64 seqno = TS_STATS_CHUNK_METADATA_GET_SEQNO(state); + if (seqno < candidate_seqno) + { + candidate_seqno = seqno; + candidate_idx = (int32) idx; + } + } + } + + /* if we found a candidate, try to evict it */ + if (candidate_idx != TS_STATS_INVALID_IDX) + { + TsStatsChunkMetadata *meta = &meta_base[candidate_idx]; + uint64 state = pg_atomic_read_u64(&meta->state); + + /* try to evict this slot, and it will update its state to in-progress + * and the bucket's last update sequence number. + */ + int32 updated_idx = ts_stats_chunk_update_at(seg, relids, candidate_idx, state); + + if (updated_idx != TS_STATS_INVALID_IDX) + { + TsStatsChunk *slot = &slot_base[updated_idx]; + ts_stats_chunk_init_slot(slot); + return updated_idx; + } + } + + return TS_STATS_INVALID_IDX; +} + +void +ts_stats_chunk_segment_finish_upsert(TsStatsChunkSegment *seg, int32 slot_idx, Oid compressed_relid) +{ + Assert(seg != NULL); + if (seg == NULL || slot_idx == TS_STATS_INVALID_IDX || !OidIsValid(compressed_relid)) + { + return; + } + + /* get the metadata pointer */ + TsStatsChunkMetadata *meta = ts_stats_chunk_metadata(seg) + slot_idx; + + /* don't do anything if the compressed_relid doesn't match */ + if (meta->relids.compressed_relid != compressed_relid) + { + return; + } + + /* get the current flag */ + uint64 state = pg_atomic_read_u64(&meta->state); + + /* only update if the slot is in the in-progress state and still valid */ + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state) && TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* clear the in-progress flag */ + uint64 new_state = (state & ~TS_STATS_CHUNK_METADATA_IN_PROGRESS_FLAG); + + /* we could do compare-exchange here, but we have no means to handle failure + * and we don't want to wait or retry, so better to simply overwrite it */ + pg_atomic_write_u64(&meta->state, new_state); + } +} + +int32 +ts_stats_chunk_segment_lookup(TsStatsChunkSegment *seg, Oid compressed_relid) +{ + Assert(seg != NULL); + if (seg == NULL || !OidIsValid(compressed_relid)) + { + return TS_STATS_INVALID_IDX; + } + + /* get the base pointers */ + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + + /* get the home location within the buckets */ + uint32 home = hash_relid(seg, compressed_relid); + uint32 bucket_size = seg->num_slots / TS_STATS_BUCKETS; + + /* sort the buckets by the last update sequence number in descending order */ + uint8 bucket_order[TS_STATS_BUCKETS]; + get_sorted_bucket_in_desc_order(seg, bucket_order); + + /* iterate over the buckets in most-recently-updated order */ + for (uint8 i = 0; i < TS_STATS_BUCKETS; i++) + { + uint8 bucket_idx = bucket_order[i]; + uint32 start = bucket_idx * bucket_size; + + /* linear probe within the bucket */ + for (uint32 j = 0; j < TS_STATS_PROBE_WIDTH; j++) + { + uint32 idx = start + ((home + j) % bucket_size); + TsStatsChunkMetadata *meta = &meta_base[idx]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* even if the current slot was evicted, we still need to check + * the all remaining slots in the bucket and all buckets until we + * exhausted the all, because the eviction may have removed old + * entried in the middle of the search window + */ + continue; + } + if (meta->relids.compressed_relid == compressed_relid) + { + return (int32) idx; + } + } + } + return TS_STATS_INVALID_IDX; +} diff --git a/src/ts_stats/ts_stats_segment.h b/src/ts_stats/ts_stats_segment.h new file mode 100644 index 00000000000..5ed3e106236 --- /dev/null +++ b/src/ts_stats/ts_stats_segment.h @@ -0,0 +1,55 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ +#pragma once + +#include "ts_stats_defs.h" + +/* + * Returns the chunk statistics segment for the current database, creating it + * lazily on first call. Returns NULL if statistics collection is disabled + * (max_chunks == 0), if DSM allocation fails, or if a published segment is + * found with an unexpected magic, i.e. not initialized. + * + * The returned pointer is cached per-backend. + */ +TsStatsChunkSegment *ts_get_stats_chunk_segment(void); + +/* + * Logical reset — clears all cached chunks WITHOUT touching the segment's + * structural fields. + */ +void ts_stats_chunk_segment_reset(TsStatsChunkSegment *seg); + +/* Try to find the location of the next available slot for upsert. + * - if the chunk already exists, return its slot index, and sets the + * in-progress flag for the slot + * - if the chunk doesn't exists, find try to find an empty slot for it and + * sets the in-progress flag for the slot + * - if there are no empty slots, find the oldest candidate in the oldest + * bucket and evict it by updating the slot's compressed_relid and setting + * the in-progress flag for the slot + * - the search is limited to the search window (TS_STATS_PROBE_WIDTH slots) in each + * bucket, and the buckets are traversed in recency order. if all slots are + * in-progress in the window, the function returns TS_STATS_INVALID_IDX + * + * Note that this call updates: + * - the in-progress flag of the slot, as mentioned above + * - the sequence number of the slot, which by itself should prevent eviction + * - the last update sequence for the bucket, which affects the recency order of the buckets + */ +int32 ts_stats_chunk_segment_prepare_upsert(TsStatsChunkSegment *seg, TsStatsRelids relids); + +/* Once the caller prepared the slot for upsert and updated the stats, it should call this + * function to clear the in-progress flag and thus make the slot eligible for eviction and + * communicate to the readers that the slot can be read consistently. + */ +void ts_stats_chunk_segment_finish_upsert(TsStatsChunkSegment *seg, int32 slot_idx, + Oid compressed_relid); + +/* Lookup — find an existing slot without allocating a new one. Return the global position of the + * slot within all slots, ignoring buckets. Returns TS_STATS_INVALID_IDX on miss. + */ +int32 ts_stats_chunk_segment_lookup(TsStatsChunkSegment *seg, Oid compressed_relid); diff --git a/src/ts_stats/ts_stats_srf.c b/src/ts_stats/ts_stats_srf.c new file mode 100644 index 00000000000..087a071a115 --- /dev/null +++ b/src/ts_stats/ts_stats_srf.c @@ -0,0 +1,437 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#include "ts_stats_defs.h" +#include "ts_stats_segment.h" + +#include +#include +#include +#include +#include +#include + +/* + * SRF Pushdown args: + * $1 compressed_relid regclass — point lookup by hash key; emits 0 or 1 row. + * $2 uncompressed_relid regclass — linear scan, or cross-checks $1. + * $3 since timestamptz — recency walk; LRU short-circuit, O(matches). + * All NULL → full scan. If both relid args are set, both must agree. + */ + +TS_FUNCTION_INFO_V1(ts_stats_chunks); +TS_FUNCTION_INFO_V1(ts_stats_reset); + +/* + * 2 relids + 2 counts (batch, column) + 9 compression min/max/sum + * (batch_rows, batch_bytes, column_bytes) + 3 sqsum (one per MinMaxSumData) + * + 10 cmd_totals + 10 cmd_last_op + 4 n_* + 2 timestamps = 42. + */ +#define STATS_CHUNKS_NCOLS 42 +#define ARG_COMPRESSED_RELID 0 +#define ARG_UNCOMPRESSED_RELID 1 +#define ARG_SINCE 2 + +Datum +ts_stats_chunks(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + InitMaterializedSRF(fcinfo, 0); + TupleDesc tupdesc = rsinfo->setDesc; + Tuplestorestate *tupstore = rsinfo->setResult; + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + PG_RETURN_VOID(); + } + + bool comp_filter = !PG_ARGISNULL(ARG_COMPRESSED_RELID); + bool uncomp_filter = !PG_ARGISNULL(ARG_UNCOMPRESSED_RELID); + Oid comp_oid = comp_filter ? PG_GETARG_OID(ARG_COMPRESSED_RELID) : InvalidOid; + Oid uncomp_oid = uncomp_filter ? PG_GETARG_OID(ARG_UNCOMPRESSED_RELID) : InvalidOid; + + if (comp_filter && !OidIsValid(comp_oid)) + { + /* don't return any rows for invalid Oid */ + PG_RETURN_VOID(); + } + + if (uncomp_filter && !OidIsValid(uncomp_oid)) + { + /* don't return any rows for invalid Oid */ + PG_RETURN_VOID(); + } + + bool since_filter = !PG_ARGISNULL(ARG_SINCE); + uint64 since_offset = 0; + + /* base_timestamp is set once at init */ + TimestampTz since_ts = since_filter ? PG_GETARG_TIMESTAMPTZ(ARG_SINCE) : 0; + if (since_filter) + { + if (since_ts <= seg->base_timestamp) + { + /* if threshold is earlier than the base, it means all record matches, so we can disable + * the filter */ + since_filter = false; + } + else + { + since_offset = (uint64) (since_ts - seg->base_timestamp); + } + } + + /* First snapshot the values */ + TsStatsChunk *slot_snap = NULL; + TsStatsChunkMetadata *meta_snap = NULL; + bool *retry_map = NULL; + uint32 n_retry_entries = 0; + uint32 first_to_try = seg->num_slots - 1; + uint32 last_to_try = 0; + uint32 snap_count = 0; + TimestampTz base = 0; + + base = seg->base_timestamp; + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + + if (comp_filter) + { + /* Point lookup based on compressed_relid */ + int32 slot_idx = ts_stats_chunk_segment_lookup(seg, comp_oid); + if (slot_idx == TS_STATS_INVALID_IDX) + { + /* no matching record found, return empty set */ + PG_RETURN_VOID(); + } + + TsStatsChunkMetadata *meta = &meta_base[slot_idx]; + TsStatsChunk *slot = &slot_base[slot_idx]; + + if (uncomp_filter) + { + /* if uncompressed_relid is also provided, check the uncompressed_relid matches as well + */ + if (meta->relids.uncompressed_relid != uncomp_oid) + { + /* no matching record found, return empty set */ + PG_RETURN_VOID(); + } + } + + if (since_filter) + { + uint64 last_update_us = slot->last_update_us; + if (last_update_us == 0 || last_update_us < since_offset) + { + /* record is older than the threshold, return empty set */ + PG_RETURN_VOID(); + } + } + + uint64 state = pg_atomic_read_u64(&meta->state); + if (TS_STATS_CHUNK_METADATA_IS_VALID(state) == 0) + { + /* slot is not valid anymore, return empty set */ + PG_RETURN_VOID(); + } + + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* slot is being updated, mark it for retry */ + retry_map = palloc0(sizeof(bool) * seg->num_slots); + retry_map[slot_idx] = true; + n_retry_entries = 1; + first_to_try = last_to_try = slot_idx; + } + else + { + /* safe to read, take a snapshot */ + slot_snap = palloc(sizeof(TsStatsChunk)); + meta_snap = palloc(sizeof(TsStatsChunkMetadata)); + memcpy(slot_snap, slot, sizeof(TsStatsChunk)); + memcpy(meta_snap, meta, sizeof(TsStatsChunkMetadata)); + snap_count = 1; + } + } + else if (uncomp_filter) + { + /* Linear scan of the metadata matching uncompressed_relid, but only one */ + slot_snap = palloc(sizeof(TsStatsChunk)); + meta_snap = palloc(sizeof(TsStatsChunkMetadata)); + retry_map = palloc0(sizeof(bool) * seg->num_slots); + + for (uint32 i = 0; i < seg->num_slots; i++) + { + TsStatsChunk *slot = &slot_base[i]; + TsStatsChunkMetadata *meta = &meta_base[i]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* don't deal with evicted/empty entries */ + continue; + } + if (meta->relids.uncompressed_relid != uncomp_oid) + { + continue; + } + /* the update timestamp is read without checking the in-progress flag + * because its meaning doesn't depend on other fields and it is consistent + * as it is. the worst case is that we might miss an update here, but + * the in-progress flag is about the overall state of the chunk slot + */ + if (since_filter && slot->last_update_us < since_offset) + { + continue; + } + + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* slot is being updated, mark it for retry */ + retry_map[i] = true; + ++n_retry_entries; + if (i < first_to_try) + { + first_to_try = i; + } + if (i > last_to_try) + { + last_to_try = i; + } + } + else + { + /* safe to read, take a snapshot */ + memcpy(slot_snap, slot, sizeof(TsStatsChunk)); + memcpy(meta_snap, meta, sizeof(TsStatsChunkMetadata)); + snap_count = 1; + } + /* either we did the snapshot or marked it for retry, in both + * cases we can stop the loop. */ + break; + } + } + else + { + /* full scan in slot order. check since timestamp if needed. */ + slot_snap = palloc(sizeof(TsStatsChunk) * seg->num_slots); + meta_snap = palloc(sizeof(TsStatsChunkMetadata) * seg->num_slots); + retry_map = palloc0(sizeof(bool) * seg->num_slots); + + for (uint32 i = 0; i < seg->num_slots; i++) + { + TsStatsChunk *slot = &slot_base[i]; + TsStatsChunkMetadata *meta = &meta_base[i]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* don't deal with evicted/empty entries */ + continue; + } + /* the update timestamp is read without checking the in-progress flag + * because its meaning doesn't depend on other fields and it is consistent + * as it is. the worst case is that we might miss an update here, but + * the in-progress flag is about the overall state of the chunk slot + */ + if (since_filter && slot->last_update_us < since_offset) + { + continue; + } + + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* slot is being updated, mark it for retry */ + retry_map[i] = true; + ++n_retry_entries; + if (i < first_to_try) + { + first_to_try = i; + } + if (i > last_to_try) + { + last_to_try = i; + } + } + else + { + /* safe to read, take a snapshot */ + memcpy(&slot_snap[snap_count], slot, sizeof(TsStatsChunk)); + memcpy(&meta_snap[snap_count], meta, sizeof(TsStatsChunkMetadata)); + snap_count++; + } + } + } + + /* give three attempts to retry the in-progress entries, use + * first_to_try and last_to_try to limit the range + */ + for (uint8 r = 0; r < 3 && n_retry_entries > 0; r++) + { + for (uint32 i = first_to_try; i <= last_to_try; i++) + { + if (!retry_map[i]) + { + /* this entry is not marked for retry, skip it */ + continue; + } + + TsStatsChunk *slot = &slot_base[i]; + TsStatsChunkMetadata *meta = &meta_base[i]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* still in progress, keep it in the retry map for the next round */ + continue; + } + + /* no longer in progress, take a snapshot and remove it from the retry map */ + memcpy(&slot_snap[snap_count], slot, sizeof(TsStatsChunk)); + memcpy(&meta_snap[snap_count], meta, sizeof(TsStatsChunkMetadata)); + snap_count++; + retry_map[i] = false; + n_retry_entries--; + } + } + + /* Build the result tuples */ + Datum values[STATS_CHUNKS_NCOLS]; + bool nulls[STATS_CHUNKS_NCOLS] = { false }; + + for (uint32 i = 0; i < snap_count; i++) + { + TsStatsChunk *s = &slot_snap[i]; + TsStatsChunkMetadata *m = &meta_snap[i]; + int c = 0; + + values[c++] = ObjectIdGetDatum(m->relids.compressed_relid); + values[c++] = ObjectIdGetDatum(m->relids.uncompressed_relid); + + /* Compression aggregates */ + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_count)); + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_block_count)); + + if (pg_atomic_read_u32(&s->compressed_batch_count) == 0) + { + for (int j = 0; j < 8; j++) + { + nulls[c + j] = true; + } + c += 8; + } + else + { + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_rows.min_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_rows.max_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u64(&s->compressed_batch_rows.sum_val)); + values[c++] = Float8GetDatum(mms_sqsum(&s->compressed_batch_rows)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_bytes.min_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_bytes.max_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u64(&s->compressed_batch_bytes.sum_val)); + values[c++] = Float8GetDatum(mms_sqsum(&s->compressed_batch_bytes)); + } + + if (pg_atomic_read_u32(&s->compressed_block_count) == 0) + { + for (int j = 0; j < 4; j++) + { + nulls[c + j] = true; + } + c += 4; + } + else + { + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_block_bytes.min_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_block_bytes.max_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u64(&s->compressed_block_bytes.sum_val)); + values[c++] = Float8GetDatum(mms_sqsum(&s->compressed_block_bytes)); + } + + /* Cmd totals */ + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_deleted)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_decompressed)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.tuples_decompressed)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_scanned)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_checked_by_bloom)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_pruned_by_bloom)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_without_bloom)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_bloom_false_positives)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_filtered_compressed)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_filtered_decompressed)); + + /* Last cmd stats */ + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_deleted)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_decompressed)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.tuples_decompressed)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_scanned)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_checked_by_bloom)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_pruned_by_bloom)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_without_bloom)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_bloom_false_positives)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_filtered_compressed)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_filtered_decompressed)); + + /* Operation counts */ + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->n_selects)); + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->n_inserts)); + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->n_updates)); + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->n_deletes)); + + /* Timestamps */ + values[c++] = TimestampTzGetDatum(base + (TimestampTz) s->first_update_us); + values[c++] = TimestampTzGetDatum(base + (TimestampTz) s->last_update_us); + + Assert(c == STATS_CHUNKS_NCOLS); + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + /* Reset nulls for the next iteration. */ + for (int j = 0; j < STATS_CHUNKS_NCOLS; j++) + { + nulls[j] = false; + } + } + + /* free snapshots */ + if (slot_snap != NULL) + { + pfree(slot_snap); + } + if (meta_snap != NULL) + { + pfree(meta_snap); + } + PG_RETURN_VOID(); +} + +Datum +ts_stats_reset(PG_FUNCTION_ARGS) +{ + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg != NULL) + { + ts_stats_chunk_segment_reset(seg); + } + PG_RETURN_VOID(); +} diff --git a/test/expected/c_unit_tests.out b/test/expected/c_unit_tests.out index 939302f0d05..dd62af76c57 100644 --- a/test/expected/c_unit_tests.out +++ b/test/expected/c_unit_tests.out @@ -16,6 +16,8 @@ CREATE OR REPLACE FUNCTION test.jsonb_utils() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_jsonb_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.compression_settings() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_compression_settings' LANGUAGE C; +CREATE OR REPLACE FUNCTION test.chunk_stats() RETURNS VOID +AS :MODULE_PATHNAME, 'ts_test_chunk_stats' LANGUAGE C; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT test.time_to_internal_conversion(); time_to_internal_conversion @@ -52,3 +54,9 @@ SELECT test.compression_settings(); ---------------------- +SELECT test.chunk_stats(); +NOTICE: Eviction test completed: 9216 evictions, 1024 slots, average lifespan 1023.53, lifespan stddev 167.91, min lifespan 138, max lifespan 1975 + chunk_stats +------------- + + diff --git a/test/expected/observ_funcs.out b/test/expected/observ_funcs.out new file mode 100644 index 00000000000..b0e993583bb --- /dev/null +++ b/test/expected/observ_funcs.out @@ -0,0 +1,55 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. +-- Smoke tests. +-- Empty cache returns no rows. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(); + count +------- + 0 + +-- Pushdown: invalid OID returns no rows (cannot match any cached chunk). +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(compressed_relid => 0::regclass); + count +------- + 0 + +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(uncompressed_relid => 0::regclass); + count +------- + 0 + +-- Pushdown: non-existent OID returns no rows. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(compressed_relid => 1::regclass); + count +------- + 0 + +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(uncompressed_relid => 1::regclass); + count +------- + 0 + +-- Pushdown: far-future since returns no rows even if the cache is non-empty. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(since => '2999-01-01'::timestamptz); + count +------- + 0 + +-- Reset is callable; cache stays empty. +SELECT _timescaledb_functions.chunk_statistics_reset(); + chunk_statistics_reset +------------------------ + + +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(); + count +------- + 0 + +-- The view exists and has the expected shape (column count is enough for a smoke). +SELECT count(*) FROM timescaledb_information.stat_chunk_activity; + count +------- + 0 + diff --git a/test/expected/pg_dump.out b/test/expected/pg_dump.out index 0382dacd2a0..ba68eff13e0 100644 --- a/test/expected/pg_dump.out +++ b/test/expected/pg_dump.out @@ -454,6 +454,7 @@ WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND timescaledb_information.job_history timescaledb_information.job_stats timescaledb_information.jobs + timescaledb_information.stat_chunk_activity -- Make sure we can't run our restoring functions as a normal perm user as that would disable functionality for the whole db \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER diff --git a/test/sql/CMakeLists.txt b/test/sql/CMakeLists.txt index 6c0562d1cd8..020c1247438 100644 --- a/test/sql/CMakeLists.txt +++ b/test/sql/CMakeLists.txt @@ -39,6 +39,7 @@ set(TEST_FILES insert_single.sql lateral.sql merge.sql + observ_funcs.sql partition.sql partition_coercion.sql partitioning.sql diff --git a/test/sql/c_unit_tests.sql b/test/sql/c_unit_tests.sql index e2a9d33c069..d2fe4a8fa39 100644 --- a/test/sql/c_unit_tests.sql +++ b/test/sql/c_unit_tests.sql @@ -24,6 +24,9 @@ AS :MODULE_PATHNAME, 'ts_test_jsonb_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.compression_settings() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_compression_settings' LANGUAGE C; +CREATE OR REPLACE FUNCTION test.chunk_stats() RETURNS VOID +AS :MODULE_PATHNAME, 'ts_test_chunk_stats' LANGUAGE C; + SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT test.time_to_internal_conversion(); @@ -33,4 +36,5 @@ SELECT test.time_utils(); SELECT test.bmslist_utils(); SELECT test.jsonb_utils(); SELECT test.compression_settings(); +SELECT test.chunk_stats(); diff --git a/test/sql/observ_funcs.sql b/test/sql/observ_funcs.sql new file mode 100644 index 00000000000..834b53ad34d --- /dev/null +++ b/test/sql/observ_funcs.sql @@ -0,0 +1,26 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +-- Smoke tests. + +-- Empty cache returns no rows. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(); + +-- Pushdown: invalid OID returns no rows (cannot match any cached chunk). +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(compressed_relid => 0::regclass); +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(uncompressed_relid => 0::regclass); + +-- Pushdown: non-existent OID returns no rows. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(compressed_relid => 1::regclass); +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(uncompressed_relid => 1::regclass); + +-- Pushdown: far-future since returns no rows even if the cache is non-empty. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(since => '2999-01-01'::timestamptz); + +-- Reset is callable; cache stays empty. +SELECT _timescaledb_functions.chunk_statistics_reset(); +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(); + +-- The view exists and has the expected shape (column count is enough for a smoke). +SELECT count(*) FROM timescaledb_information.stat_chunk_activity; diff --git a/test/src/CMakeLists.txt b/test/src/CMakeLists.txt index c1ef2691592..12c428462df 100644 --- a/test/src/CMakeLists.txt +++ b/test/src/CMakeLists.txt @@ -5,6 +5,7 @@ set(SOURCES test_compression_settings.c test_bmslist_utils.c test_jsonb_utils.c + test_stats.c test_scanner.c test_time_to_internal.c test_time_utils.c diff --git a/test/src/test_stats.c b/test/src/test_stats.c new file mode 100644 index 00000000000..3494ed151fd --- /dev/null +++ b/test/src/test_stats.c @@ -0,0 +1,139 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +/* + * Unit tests for the chunk-stats for the parts that are hard to + * reach from SQL. + * + * The DSM/shmem layer is bypassed: we palloc a tiny segment in regular + * backend memory, set its structural fields. + */ + +#include +#include +#include + +#include "test_utils.h" + +#include "ts_stats/ts_stats_defs.h" +#include "ts_stats/ts_stats_segment.h" + +static TsStatsChunkSegment * +make_test_segment(uint32 num_slots) +{ + TsStatsChunkSegment *seg = palloc0(ts_stats_chunk_segment_size(num_slots)); + seg->magic = TS_STATS_SHMEM_MAGIC; + seg->dboid = MyDatabaseId; + seg->base_timestamp = GetCurrentTimestamp(); + seg->num_slots = num_slots; + return seg; +} + +static void +test_simple_upsert_lookup() +{ + TsStatsChunkSegment *seg = make_test_segment(16); + TsStatsRelids relids = { + .compressed_relid = 12345, + .uncompressed_relid = 54321, + }; + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + Assert(slot_idx != TS_STATS_INVALID_IDX && "Failed to prepare upsert"); + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); + Assert(ts_stats_chunk_segment_lookup(seg, relids.compressed_relid) == slot_idx && + "Lookup index does not match upsert index"); + pfree(seg); +} + +static int32 +insert_one_chunk(TsStatsChunkSegment *seg, TsStatsRelids relids) +{ + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + Assert(slot_idx != TS_STATS_INVALID_IDX && "Failed to prepare upsert"); + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); + return slot_idx; +} + +static void +test_eviction() +{ +#define NUM_SLOTS 1024 +#define MIN_LIFESPAN (NUM_SLOTS / TS_STATS_PROBE_WIDTH / TS_STATS_BUCKETS) + TsStatsChunkSegment *seg = make_test_segment(NUM_SLOTS); + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + Oid base_compressed_relid = 10000; + Oid uncompressed_relid = 54321; + int32 slot_indices[NUM_SLOTS] = { 0 }; + uint64 slot_seqno[NUM_SLOTS] = { 0 }; + uint64 sum_lifespan = 0; + double sumsq_lifespan = 0; + int32 num_evictions = 0; + uint64 min_lifespan = UINT64_MAX; + uint64 max_lifespan = 0; + + for (int j = 0; j < 10; j++) + { + for (int i = 0; i < NUM_SLOTS; i++) + { + /* + * the i,j makes sure we don't always hit the same slots which would favour the + * eviction policy. + */ + TsStatsRelids relids = { + .compressed_relid = base_compressed_relid + i + (j * 11 * NUM_SLOTS), + .uncompressed_relid = uncompressed_relid, + }; + int32 slot_idx = insert_one_chunk(seg, relids); + Assert(slot_idx != TS_STATS_INVALID_IDX && "Failed to prepare upsert"); + Assert(slot_idx < NUM_SLOTS && + "Slot index exceeds number of slots, eviction should have happened"); + if (slot_indices[slot_idx] != 0) + { + uint64 old_seqno = slot_seqno[slot_idx]; + uint64 new_seqno = pg_atomic_read_u64(&meta_base[slot_idx].state) >> + TS_STATS_CHUNK_METADATA_SEQNO_SHIFT; + uint64 lifespan = new_seqno - old_seqno; + sum_lifespan += lifespan; + sumsq_lifespan += (double) lifespan * lifespan; + num_evictions++; + if (lifespan < min_lifespan) + { + min_lifespan = lifespan; + } + if (lifespan > max_lifespan) + { + max_lifespan = lifespan; + } + Assert(new_seqno > (old_seqno + MIN_LIFESPAN) && + "Sequence number did not increase as expected after eviction"); + } + slot_indices[slot_idx] = relids.compressed_relid; + slot_seqno[slot_idx] = pg_atomic_read_u64(&meta_base[slot_idx].state) >> + TS_STATS_CHUNK_METADATA_SEQNO_SHIFT; + Assert(ts_stats_chunk_segment_lookup(seg, relids.compressed_relid) == slot_idx && + "Lookup index does not match upsert index"); + } + } + + elog(NOTICE, + "Eviction test completed: %d evictions, %d slots, average lifespan %.2f, lifespan stddev " + "%.2f, min lifespan %d, max lifespan %d", + num_evictions, + NUM_SLOTS, + (double) sum_lifespan / num_evictions, + sqrt((sumsq_lifespan / num_evictions) - + ((sum_lifespan / num_evictions) * (sum_lifespan / num_evictions))), + (int) min_lifespan, + (int) max_lifespan); + pfree(seg); +} + +TS_TEST_FN(ts_test_chunk_stats) +{ + test_simple_upsert_lookup(); + test_eviction(); + PG_RETURN_VOID(); +} diff --git a/tsl/src/chunk_split.c b/tsl/src/chunk_split.c index f7261f33362..3267064cb77 100644 --- a/tsl/src/chunk_split.c +++ b/tsl/src/chunk_split.c @@ -416,8 +416,10 @@ route_next_compressed_tuple(TupleTableSlot *slot, SplitContext *scontext, int *r tuple = ExecFetchSlotHeapTuple(slot, false, NULL); - RowDecompressor decompressor = - build_decompressor(slot->tts_tupleDescriptor, csp->noncompressed_tupdesc); + RowDecompressor decompressor = build_decompressor(slot->tts_tupleDescriptor, + csp->noncompressed_tupdesc, + csettings->fd.compress_relid, + csettings->fd.relid); heap_deform_tuple(tuple, decompressor.in_desc, diff --git a/tsl/src/compression/compression.c b/tsl/src/compression/compression.c index 157a9babe8d..2de4dc54e82 100644 --- a/tsl/src/compression/compression.c +++ b/tsl/src/compression/compression.c @@ -48,6 +48,7 @@ #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_settings.h" +#include "ts_stats/ts_stats_record.h" #include /* @@ -1352,6 +1353,9 @@ row_compressor_init(RowCompressor *row_compressor, const CompressionSettings *se check_for_limited_size_compressors(row_compressor->per_column, row_compressor->n_input_columns); + row_compressor->cached_relids.compressed_relid = settings->fd.compress_relid; + row_compressor->cached_relids.uncompressed_relid = settings->fd.relid; + ts_stats_compression_acc_init(&row_compressor->observ_acc); MemoryContextSwitchTo(old_context); } @@ -1630,6 +1634,9 @@ row_compressor_build_tuple(RowCompressor *row_compressor) { row_compressor->compressed_values[compressed_col] = PointerGetDatum(compressed_data); + + ts_stats_compression_acc_column(&row_compressor->observ_acc, + VARSIZE_ANY_EXHDR(compressed_data)); } } else if (column->segment_info != NULL) @@ -1788,6 +1795,9 @@ row_compressor_flush(RowCompressor *row_compressor, BulkWriter *writer, bool cha row_compressor->rows_compressed_into_current_value); } + ts_stats_compression_acc_batch(&row_compressor->observ_acc, + row_compressor->rows_compressed_into_current_value); + MemoryContextSwitchTo(old_cxt); row_compressor_clear_batch(row_compressor, changed_groups); } @@ -1801,6 +1811,9 @@ row_compressor_reset(RowCompressor *row_compressor) void row_compressor_close(RowCompressor *row_compressor) { + /* Flush the observability data */ + ts_stats_chunk_record_compression(row_compressor->cached_relids, &row_compressor->observ_acc); + pfree(row_compressor->compressed_is_null); pfree(row_compressor->compressed_values); pfree(row_compressor->per_column); @@ -2006,7 +2019,7 @@ bulk_writer_close(BulkWriter *writer) **********************/ RowDecompressor -build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc) +build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc, Oid in_oid, Oid out_oid) { AttrNumber count_meta_attnum = InvalidAttrNumber; AttrMap *attrmap = build_decompress_attrmap(out_desc, in_desc, &count_meta_attnum); @@ -2049,9 +2062,21 @@ build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc) detoaster_init(&decompressor.detoaster, CurrentMemoryContext); + row_decompressor_init_stats(&decompressor, in_oid, out_oid, CMD_SELECT); + return decompressor; } +void +row_decompressor_init_stats(RowDecompressor *decompressor, Oid compressed_relid, + Oid uncompressed_relid, CmdType cmd_type) +{ + memset(&decompressor->observ_counters, 0, sizeof(decompressor->observ_counters)); + decompressor->cached_relids.compressed_relid = compressed_relid; + decompressor->cached_relids.uncompressed_relid = uncompressed_relid; + decompressor->cmd_type = cmd_type; +} + void row_decompressor_reset(RowDecompressor *decompressor) { @@ -2061,9 +2086,22 @@ row_decompressor_reset(RowDecompressor *decompressor) decompressor->tuples_decompressed = 0; } +void +row_decompressor_flush_stats(RowDecompressor *decompressor) +{ + /* Flush the observability data */ + ts_stats_chunk_record_cmd(decompressor->cached_relids, + decompressor->cmd_type, + &decompressor->observ_counters); + /* Reset the cached relids */ + decompressor->cached_relids.compressed_relid = InvalidOid; + decompressor->cached_relids.uncompressed_relid = InvalidOid; +} + void row_decompressor_close(RowDecompressor *decompressor) { + row_decompressor_flush_stats(decompressor); MemoryContextDelete(decompressor->per_compressed_row_ctx); detoaster_close(&decompressor->detoaster); free_attrmap(decompressor->attrmap); @@ -2098,8 +2136,10 @@ decompress_chunk(Oid in_table, Oid out_table) PushActiveSnapshot(GetLatestSnapshot()); BulkWriter writer = bulk_writer_build(out_rel, 0); - RowDecompressor decompressor = - build_decompressor(RelationGetDescr(in_rel), RelationGetDescr(out_rel)); + RowDecompressor decompressor = build_decompressor(RelationGetDescr(in_rel), + RelationGetDescr(out_rel), + RelationGetRelid(in_rel), + RelationGetRelid(out_rel)); TupleTableSlot *slot = table_slot_create(in_rel, NULL); TableScanDesc scan = table_beginscan(in_rel, GetActiveSnapshot(), 0, (ScanKey) NULL); int64 report_reltuples = calculate_reltuples_to_report(in_rel->rd_rel->reltuples); diff --git a/tsl/src/compression/compression.h b/tsl/src/compression/compression.h index 11b5c0e7258..4121d2dff9c 100644 --- a/tsl/src/compression/compression.h +++ b/tsl/src/compression/compression.h @@ -6,6 +6,7 @@ #pragma once #include +#include "ts_stats/ts_stats_defs.h" #include #include #include @@ -20,6 +21,7 @@ typedef struct BulkInsertStateData *BulkInsertState; #include "hypertable.h" #include "nodes/columnar_scan/detoaster.h" #include "ts_catalog/compression_settings.h" +#include "ts_stats/ts_stats_record.h" /* * Compressed data starts with a specialized varlen type starting with the usual @@ -179,6 +181,10 @@ typedef struct RowDecompressor AttrMap *attrmap; Detoaster detoaster; + + TsStatsRelids cached_relids; + CmdType cmd_type; + SharedCounters observ_counters; } RowDecompressor; /* @@ -306,6 +312,10 @@ typedef struct RowCompressor bool needs_analyze_segmentby; List *metadata_builders; /* List of BatchMetadataBuilder */ + + TsStatsRelids cached_relids; + CompressionStatsAccumulator observ_acc; + } RowCompressor; /* @@ -429,10 +439,14 @@ extern void segment_info_update(SegmentInfo *segment_info, Datum val, bool is_nu extern BulkWriter bulk_writer_build(Relation out_rel, int insert_options); extern BulkWriter *bulk_writer_alloc(Relation out_rel, int insert_options); extern void bulk_writer_close(BulkWriter *writer); -extern RowDecompressor build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc); +extern RowDecompressor build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc, + Oid in_oid, Oid out_oid); extern void row_decompressor_reset(RowDecompressor *decompressor); extern void row_decompressor_close(RowDecompressor *decompressor); +extern void row_decompressor_init_stats(RowDecompressor *decompressor, Oid compressed_relid, + Oid uncompressed_relid, CmdType cmd_type); +extern void row_decompressor_flush_stats(RowDecompressor *decompressor); extern int decompress_batch(RowDecompressor *decompressor); extern bool decompress_batch_next_row(RowDecompressor *decompressor, AttrNumber *attnos, int num_attnos); diff --git a/tsl/src/compression/compression_dml.c b/tsl/src/compression/compression_dml.c index 298f97b3005..4dbb655fd34 100644 --- a/tsl/src/compression/compression_dml.c +++ b/tsl/src/compression/compression_dml.c @@ -22,6 +22,7 @@ #include #include "foreach_ptr.h" +#include "ts_stats/ts_stats_record.h" #include #include #include @@ -60,7 +61,7 @@ static struct decompress_batches_stats decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, Snapshot snapshot, bool *skip_current_tuple, bool delete_only, List *is_nulls, InvalidationContext *invalidation_ctx, CachedDecompressionState *cdst, - TupleTableSlot *insert_slot); + TupleTableSlot *insert_slot, CmdType cmd_type); static BatchQualSummary batch_matches(RowDecompressor *decompressor, ScanKeyData *scankeys, int num_scankeys, tuple_filtering_constraints *constraints, @@ -92,6 +93,8 @@ static bool can_vectorize_constraint_checks(tuple_filtering_constraints *constra static void update_scankeys(ScanKeyWithAttnos *scankeys, TupleTableSlot *slot, int null_flags); static void init_upsert_bloom_state(ChunkInsertState *cis); static Bitmapset *get_arbiter_index_attnums(ChunkInsertState *cis); +static inline void copy_stats_to_observ_counters(RowDecompressor *d, + const struct decompress_batches_stats *s); static AttrNumber TupleDescGetAttrNumber(TupleDesc desc, const char *name) @@ -521,7 +524,9 @@ decompress_batches_for_insert(ChunkInsertState *cis, TupleTableSlot *slot) NIL, NULL /* no CAgg invalidation for inserts */, cdst, - slot); + slot, + CMD_INSERT); + if (index_rel) { index_close(index_rel, AccessShareLock); @@ -685,7 +690,8 @@ decompress_batches_for_update_delete(ModifyHypertableState *ht_state, Chunk *chu is_null, ht_state->has_continuous_aggregate ? &invalidation_ctx : NULL, &temp_cdst, - NULL); + NULL, + ht_state->mt->operation); /* close the selected index */ if (matching_index_rel) @@ -821,7 +827,7 @@ static struct decompress_batches_stats decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, Snapshot snapshot, bool *skip_current_tuple, bool delete_only, List *is_nulls, InvalidationContext *invalidation_ctx, CachedDecompressionState *cdst, - TupleTableSlot *insert_slot) + TupleTableSlot *insert_slot, CmdType cmd_type) { HeapTuple compressed_tuple; BulkWriter writer; @@ -845,6 +851,16 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S struct decompress_batches_stats stats = { 0 }; + /* + * initialize stats related fields of the compressor, so it can be + * used even if the decompression is skipped by the bloom filter or + * other filters + */ + row_decompressor_init_stats(&decompressor, + RelationGetRelid(in_rel), + RelationGetRelid(out_rel), + cmd_type); + /* TODO: Optimization by reusing the index scan while working on a single chunk */ if (index_rel) @@ -961,7 +977,14 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S if (!decompressor_initialized) { - decompressor = build_decompressor(RelationGetDescr(in_rel), RelationGetDescr(out_rel)); + decompressor = build_decompressor(RelationGetDescr(in_rel), + RelationGetDescr(out_rel), + RelationGetRelid(in_rel), + RelationGetRelid(out_rel)); + + /* set the cmd type in the metrics record */ + decompressor.cmd_type = cmd_type; + decompressor_initialized = true; writer = bulk_writer_build(out_rel, 0); meta_count_attno = TupleDescGetAttrNumber(decompressor.in_desc, @@ -1040,6 +1063,7 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S if (skip_current_tuple && *skip_current_tuple) { + copy_stats_to_observ_counters(&decompressor, &stats); row_decompressor_close(&decompressor); bulk_writer_close(&writer); decompress_batch_endscan(scan); @@ -1075,6 +1099,7 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S if (result != TM_Ok) { write_logical_replication_msg_decompression_end(); + copy_stats_to_observ_counters(&decompressor, &stats); row_decompressor_close(&decompressor); bulk_writer_close(&writer); decompress_batch_endscan(scan); @@ -1116,11 +1141,22 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S } ExecDropSingleTupleTableSlot(slot); decompress_batch_endscan(scan); + copy_stats_to_observ_counters(&decompressor, &stats); + if (decompressor_initialized) { row_decompressor_close(&decompressor); bulk_writer_close(&writer); } + else + { + /* + * we accumulate stats in the decompressor even if + * skip decompression because of other filters like + * the bloom, null or index filter. + */ + row_decompressor_flush_stats(&decompressor); + } if (ts_guc_debug_compression_path_info) { @@ -2458,3 +2494,18 @@ can_vectorize_constraint_checks(tuple_filtering_constraints *constraints, return true; } + +static inline void +copy_stats_to_observ_counters(RowDecompressor *d, const struct decompress_batches_stats *s) +{ + d->observ_counters.batches_deleted = s->batches_deleted; + d->observ_counters.batches_decompressed = s->batches_decompressed; + d->observ_counters.tuples_decompressed = s->tuples_decompressed; + d->observ_counters.batches_scanned = s->batches_scanned; + d->observ_counters.batches_checked_by_bloom = s->batches_checked_by_bloom; + d->observ_counters.batches_pruned_by_bloom = s->batches_pruned_by_bloom; + d->observ_counters.batches_without_bloom = s->batches_without_bloom; + d->observ_counters.batches_bloom_false_positives = s->batches_bloom_false_positives; + d->observ_counters.batches_filtered_compressed = s->batches_filtered_compressed; + d->observ_counters.batches_filtered_decompressed = s->batches_filtered_decompressed; +} diff --git a/tsl/src/compression/recompress.c b/tsl/src/compression/recompress.c index 8d72369c6a6..373fa8eb3c5 100644 --- a/tsl/src/compression/recompress.c +++ b/tsl/src/compression/recompress.c @@ -334,7 +334,9 @@ recompress_chunk_segmentwise_impl(Chunk *uncompressed_chunk, /******************** row decompressor **************/ RowDecompressor decompressor = build_decompressor(RelationGetDescr(compressed_chunk_rel), - RelationGetDescr(uncompressed_chunk_rel)); + RelationGetDescr(uncompressed_chunk_rel), + RelationGetRelid(compressed_chunk_rel), + RelationGetRelid(uncompressed_chunk_rel)); /********** row compressor *******************/ RowCompressor row_compressor; @@ -769,7 +771,9 @@ perform_recompression(RecompressContext *recompress_ctx, Relation compressed_chu PushActiveSnapshot(GetTransactionSnapshot()); decompressor = build_decompressor(RelationGetDescr(compressed_chunk_rel), - RelationGetDescr(uncompressed_chunk_rel)); + RelationGetDescr(uncompressed_chunk_rel), + RelationGetRelid(compressed_chunk_rel), + RelationGetRelid(uncompressed_chunk_rel)); tuplesortstate = tuplesort_begin_heap(RelationGetDescr(uncompressed_chunk_rel), recompress_ctx->n_keys, diff --git a/tsl/src/nodes/columnar_scan/compressed_batch.c b/tsl/src/nodes/columnar_scan/compressed_batch.c index 678799738e1..c9bfa59b58f 100644 --- a/tsl/src/nodes/columnar_scan/compressed_batch.c +++ b/tsl/src/nodes/columnar_scan/compressed_batch.c @@ -195,6 +195,7 @@ decompress_column(DecompressContext *dcontext, DecompressBatchState *batch_state bool isnull; Datum value = slot_getattr(compressed_slot, column_description->compressed_scan_attno, &isnull); + dcontext->compressed_relid = compressed_slot->tts_tableOid; if (isnull) { @@ -228,6 +229,11 @@ decompress_column(DecompressContext *dcontext, DecompressBatchState *batch_state return; } + dcontext->batches_decompressed++; + /* to backfill the column's compressed bytes. */ + ts_stats_compression_acc_column(&dcontext->observ_acc, + VARSIZE_ANY_EXHDR(DatumGetPointer(value))); + /* Decompress the entire batch if it is supported. */ ArrowArray *arrow = NULL; if (dcontext->enable_bulk_decompression && column_description->bulk_decompression_supported) @@ -1031,6 +1037,9 @@ compressed_batch_set_compressed_tuple(DecompressContext *dcontext, } } + dcontext->tuples_decompressed += batch_state->total_batch_rows; + ts_stats_compression_acc_batch(&dcontext->observ_acc, batch_state->total_batch_rows); + CompressedBatchVectorQualState cbvqstate = { .vqstate = { .vectorized_quals_constified = dcontext->vectorized_quals_constified, diff --git a/tsl/src/nodes/columnar_scan/decompress_context.h b/tsl/src/nodes/columnar_scan/decompress_context.h index 37883e0cca1..2f558124ef3 100644 --- a/tsl/src/nodes/columnar_scan/decompress_context.h +++ b/tsl/src/nodes/columnar_scan/decompress_context.h @@ -14,6 +14,7 @@ #include "batch_array.h" #include "detoaster.h" +#include "ts_stats/ts_stats_record.h" typedef enum CompressionColumnType { @@ -98,6 +99,12 @@ typedef struct DecompressContext int32 chunk_status; + /* Stats */ + Oid compressed_relid; + uint64 batches_decompressed; + uint64 tuples_decompressed; + CompressionStatsAccumulator observ_acc; + } DecompressContext; #endif /* TIMESCALEDB_DECOMPRESS_CONTEXT_H */ diff --git a/tsl/src/nodes/columnar_scan/exec.c b/tsl/src/nodes/columnar_scan/exec.c index 7318fb4d742..aeec3235511 100644 --- a/tsl/src/nodes/columnar_scan/exec.c +++ b/tsl/src/nodes/columnar_scan/exec.c @@ -5,6 +5,7 @@ */ #include +#include "ts_stats/ts_stats_defs.h" #include #include #include @@ -31,6 +32,7 @@ #include "nodes/columnar_scan/exec.h" #include "nodes/columnar_scan/planner.h" #include "ts_catalog/array_utils.h" +#include "ts_stats/ts_stats_record.h" #if PG18_GE #include @@ -198,6 +200,8 @@ columnar_scan_begin(CustomScanState *node, EState *estate, int eflags) Plan *compressed_scan = linitial(cscan->custom_plans); Assert(list_length(cscan->custom_plans) == 1); + ts_stats_compression_acc_init(&dcontext->observ_acc); + PlanState *ps = &node->ss.ps; if (ps->ps_ProjInfo) { @@ -504,11 +508,24 @@ columnar_scan_end(CustomScanState *node) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; BatchQueue *bq = chunk_state->batch_queue; + DecompressContext *dcontext = &chunk_state->decompress_context; bq->funcs->free(bq); ExecEndNode(linitial(node->custom_ps)); detoaster_close(&chunk_state->decompress_context.detoaster); + + /* Finish the observability aggregates. */ + SharedCounters sc = { 0 }; + sc.tuples_decompressed = dcontext->tuples_decompressed; + sc.batches_decompressed = dcontext->batches_decompressed; + TsStatsRelids relids = { + .compressed_relid = dcontext->compressed_relid, + .uncompressed_relid = chunk_state->chunk_relid, + }; + ts_stats_chunk_record_cmd(relids, CMD_SELECT, &sc); + + ts_stats_chunk_record_decompression(relids, &dcontext->observ_acc); } /* diff --git a/tsl/test/expected/compress_observ-15.out b/tsl/test/expected/compress_observ-15.out new file mode 100644 index 00000000000..d3ea65ba857 --- /dev/null +++ b/tsl/test/expected/compress_observ-15.out @@ -0,0 +1,1395 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Test the integration of the compression and observability features +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + create_hypertable +------------------- + (1,t) + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; +\pset expanded on +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +SELECT COUNT(*) FROM t; +-[ RECORD 1 ] +count | 4900 + +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 8 +total_tuples_decompressed | 1600 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 8 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 2400 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 12 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 900 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 4 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 10 +total_tuples_decompressed | 3200 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 15 +total_tuples_decompressed | 4800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 5 +total_tuples_decompressed | 1800 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; +-[ RECORD 1 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +chunk_id | 1 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 584 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 273.740314897167 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +chunk_id | 2 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 883.073420881111 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 309.946059823318 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +chunk_id | 3 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_avg | 900 +compressed_batch_rows_stddev | 0 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_avg | 2964 +compressed_batch_bytes_stddev | 0 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_avg | 592 +compressed_block_bytes_stddev | 259.505992223687 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +-[ RECORD 1 ]----------+- +chunk_statistics_reset | + +SELECT count(*) FROM t where a % 19 = 9; +-[ RECORD 1 ] +count | 490 + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 2 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 460 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 608 +compressed_block_bytes_stddev | 148 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 3 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 324 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 609 +compressed_block_bytes_stddev | 201.787567065521 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 1 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 676 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 676 +compressed_block_bytes_stddev | 0 + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 8 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 430 +compressed_block_bytes_stddev | 168.321121669266 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 12 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 431 +compressed_block_bytes_stddev | 201.075994478594 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 4 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 482 +compressed_block_bytes_stddev | 150.983442800858 + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 1 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 9 +total_batches_checked_by_bloom | 9 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 8 +total_batches_checked_by_bloom | 8 +total_batches_pruned_by_bloom | 4 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 4 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 12 +total_batches_checked_by_bloom | 12 +total_batches_pruned_by_bloom | 6 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 6 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t; diff --git a/tsl/test/expected/compress_observ-16.out b/tsl/test/expected/compress_observ-16.out new file mode 100644 index 00000000000..d3ea65ba857 --- /dev/null +++ b/tsl/test/expected/compress_observ-16.out @@ -0,0 +1,1395 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Test the integration of the compression and observability features +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + create_hypertable +------------------- + (1,t) + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; +\pset expanded on +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +SELECT COUNT(*) FROM t; +-[ RECORD 1 ] +count | 4900 + +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 8 +total_tuples_decompressed | 1600 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 8 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 2400 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 12 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 900 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 4 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 10 +total_tuples_decompressed | 3200 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 15 +total_tuples_decompressed | 4800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 5 +total_tuples_decompressed | 1800 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; +-[ RECORD 1 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +chunk_id | 1 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 584 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 273.740314897167 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +chunk_id | 2 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 883.073420881111 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 309.946059823318 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +chunk_id | 3 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_avg | 900 +compressed_batch_rows_stddev | 0 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_avg | 2964 +compressed_batch_bytes_stddev | 0 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_avg | 592 +compressed_block_bytes_stddev | 259.505992223687 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +-[ RECORD 1 ]----------+- +chunk_statistics_reset | + +SELECT count(*) FROM t where a % 19 = 9; +-[ RECORD 1 ] +count | 490 + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 2 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 460 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 608 +compressed_block_bytes_stddev | 148 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 3 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 324 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 609 +compressed_block_bytes_stddev | 201.787567065521 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 1 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 676 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 676 +compressed_block_bytes_stddev | 0 + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 8 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 430 +compressed_block_bytes_stddev | 168.321121669266 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 12 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 431 +compressed_block_bytes_stddev | 201.075994478594 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 4 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 482 +compressed_block_bytes_stddev | 150.983442800858 + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 1 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 9 +total_batches_checked_by_bloom | 9 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 8 +total_batches_checked_by_bloom | 8 +total_batches_pruned_by_bloom | 4 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 4 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 12 +total_batches_checked_by_bloom | 12 +total_batches_pruned_by_bloom | 6 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 6 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t; diff --git a/tsl/test/expected/compress_observ-17.out b/tsl/test/expected/compress_observ-17.out new file mode 100644 index 00000000000..d3ea65ba857 --- /dev/null +++ b/tsl/test/expected/compress_observ-17.out @@ -0,0 +1,1395 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Test the integration of the compression and observability features +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + create_hypertable +------------------- + (1,t) + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; +\pset expanded on +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +SELECT COUNT(*) FROM t; +-[ RECORD 1 ] +count | 4900 + +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 8 +total_tuples_decompressed | 1600 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 8 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 2400 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 12 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 900 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 4 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 10 +total_tuples_decompressed | 3200 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 15 +total_tuples_decompressed | 4800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 5 +total_tuples_decompressed | 1800 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; +-[ RECORD 1 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +chunk_id | 1 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 584 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 273.740314897167 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +chunk_id | 2 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 883.073420881111 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 309.946059823318 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +chunk_id | 3 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_avg | 900 +compressed_batch_rows_stddev | 0 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_avg | 2964 +compressed_batch_bytes_stddev | 0 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_avg | 592 +compressed_block_bytes_stddev | 259.505992223687 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +-[ RECORD 1 ]----------+- +chunk_statistics_reset | + +SELECT count(*) FROM t where a % 19 = 9; +-[ RECORD 1 ] +count | 490 + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 2 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 460 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 608 +compressed_block_bytes_stddev | 148 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 3 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 324 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 609 +compressed_block_bytes_stddev | 201.787567065521 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 1 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 676 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 676 +compressed_block_bytes_stddev | 0 + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 8 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 430 +compressed_block_bytes_stddev | 168.321121669266 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 12 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 431 +compressed_block_bytes_stddev | 201.075994478594 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 4 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 482 +compressed_block_bytes_stddev | 150.983442800858 + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 1 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 9 +total_batches_checked_by_bloom | 9 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 8 +total_batches_checked_by_bloom | 8 +total_batches_pruned_by_bloom | 4 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 4 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 12 +total_batches_checked_by_bloom | 12 +total_batches_pruned_by_bloom | 6 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 6 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t; diff --git a/tsl/test/expected/compress_observ-18.out b/tsl/test/expected/compress_observ-18.out new file mode 100644 index 00000000000..0adc8733b16 --- /dev/null +++ b/tsl/test/expected/compress_observ-18.out @@ -0,0 +1,1395 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Test the integration of the compression and observability features +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + create_hypertable +------------------- + (1,t) + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; +\pset expanded on +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +SELECT COUNT(*) FROM t; +-[ RECORD 1 ] +count | 4900 + +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 8 +total_tuples_decompressed | 1600 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 8 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 2400 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 12 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 900 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 4 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 10 +total_tuples_decompressed | 3200 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 15 +total_tuples_decompressed | 4800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 5 +total_tuples_decompressed | 1800 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; +-[ RECORD 1 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +chunk_id | 1 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_avg | 1824 +compressed_batch_bytes_stddev | 412 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 364 +compressed_block_bytes_stddev | 180.681377014899 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +chunk_id | 2 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_avg | 1825 +compressed_batch_bytes_stddev | 580.770369614551 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 365 +compressed_block_bytes_stddev | 202.38295931778 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +chunk_id | 3 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_avg | 900 +compressed_batch_rows_stddev | 0 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_avg | 2028 +compressed_batch_bytes_stddev | 0 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_avg | 405 +compressed_block_bytes_stddev | 173.022079515881 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +-[ RECORD 1 ]----------+- +chunk_statistics_reset | + +SELECT count(*) FROM t where a % 19 = 9; +-[ RECORD 1 ] +count | 490 + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 2 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 748 +compressed_batch_bytes_avg | 374 +compressed_batch_bytes_stddev | 374 +compressed_block_bytes_min | 460 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 604 +compressed_block_bytes_stddev | 144 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 3 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 748 +compressed_batch_bytes_avg | 498 +compressed_batch_bytes_stddev | 352.610581551692 +compressed_block_bytes_min | 324 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 606 +compressed_block_bytes_stddev | 199.875516815398 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 1 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 668 +compressed_block_bytes_max | 668 +compressed_block_bytes_avg | 668 +compressed_block_bytes_stddev | 0 + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 8 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 748 +compressed_batch_bytes_avg | 374 +compressed_batch_bytes_stddev | 374 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 414 +compressed_block_bytes_stddev | 168.463645929916 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 12 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 748 +compressed_batch_bytes_avg | 498 +compressed_batch_bytes_stddev | 352.610581551692 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 414 +compressed_block_bytes_stddev | 196.533288104925 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 4 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 668 +compressed_block_bytes_avg | 460 +compressed_block_bytes_stddev | 150.412765415705 + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 1 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 9 +total_batches_checked_by_bloom | 9 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 8 +total_batches_checked_by_bloom | 8 +total_batches_pruned_by_bloom | 4 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 4 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 12 +total_batches_checked_by_bloom | 12 +total_batches_pruned_by_bloom | 6 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 6 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t; diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index 70f04d68b7d..e61952c12ba 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -40,6 +40,8 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text _timescaledb_functions.chunk_constraint_add_table_constraint(_timescaledb_catalog.chunk_constraint) _timescaledb_functions.chunk_id_from_relid(oid) _timescaledb_functions.chunk_rewrite_cleanup() + _timescaledb_functions.chunk_statistics(regclass,regclass,timestamp with time zone) + _timescaledb_functions.chunk_statistics_reset() _timescaledb_functions.chunk_status(regclass) _timescaledb_functions.chunk_status_text(integer) _timescaledb_functions.chunk_status_text(regclass) diff --git a/tsl/test/sql/.gitignore b/tsl/test/sql/.gitignore index 9ff2cc72f47..7af56556318 100644 --- a/tsl/test/sql/.gitignore +++ b/tsl/test/sql/.gitignore @@ -11,3 +11,4 @@ /plan_skip_scan_dagg-*.sql /columnar_index_scan-*.sql /vector_agg_planning-*.sql +/compress_observ-*.sql diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index 061ded82e04..c247d1289a0 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -98,6 +98,7 @@ set(TEST_TEMPLATES cagg_permissions.sql.in columnar_index_scan.sql.in compress_bloom_sparse.sql.in + compress_observ.sql.in compression_permissions.sql.in foreign_keys_test.sql.in modify_exclusion.sql.in diff --git a/tsl/test/sql/compress_observ.sql.in b/tsl/test/sql/compress_observ.sql.in new file mode 100644 index 00000000000..aad19b58956 --- /dev/null +++ b/tsl/test/sql/compress_observ.sql.in @@ -0,0 +1,202 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +-- Test the integration of the compression and observability features + +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; + +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; + +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); + +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; + +\pset expanded on + +SELECT compress_chunk(c) FROM show_chunks('t') c; + +SELECT * FROM observ; -- see view definition above + +SELECT COUNT(*) FROM t; +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; + +SELECT * FROM observ; -- see view definition above + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +SELECT count(*) FROM t where a % 19 = 9; + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t;