Stats collection for compressed chunks

.github/workflows/linux-32bit-build-and-test.yaml

@@ -70,6 +70,7 @@ jobs:
           compress_bloom_sparse_debug
           compress_compbloom_hash_pushdown
           compress_composite_bloom_debug
+          compress_observ-*
           compress_qualpushdown_saop
           compress_sort_transform
           compression_allocation

.github/workflows/windows-build-and-test.yaml

@@ -89,6 +89,7 @@ jobs:
         compress_bloom_sparse_debug
         compress_compbloom_hash_pushdown
         compress_composite_bloom_debug
+        compress_observ-*
         compress_qualpushdown_saop
         compress_sort_transform
         compression_algos

.unreleased/pr_9730

@@ -0,0 +1 @@
+Implements: #9730 In memory observability for compressed chunks

cmake/ScriptFiles.cmake

@@ -42,6 +42,7 @@ set(SOURCE_FILES
     bgw_scheduler.sql
     metadata.sql
     uuidv7.sql
+    ts_stats.sql
     views.sql
     views_experimental.sql
     gapfill.sql

sql/ts_stats.sql

@@ -0,0 +1,71 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+-- Chunk statistics. This function takes three optional parameters:
+-- - compressed_relid: if provided, returns stats for the specified compressed chunk only; otherwise, returns stats for all chunks.
+-- - uncompressed_relid: if provided, returns stats for the specified user chunk only; if compressed_relid is also provided, both must agree.
+-- - since: if provided, returns stats for operations that occurred since the specified timestamp; otherwise, returns all stats.
+--
+CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_statistics(
+    compressed_relid   regclass    DEFAULT NULL,
+    uncompressed_relid regclass    DEFAULT NULL,
+    since              timestamptz DEFAULT NULL
+)
+RETURNS TABLE (
+    compressed_relid                   oid,
+    uncompressed_relid                 oid,
+    -- Compression
+    compressed_batch_count             bigint,
+    compressed_block_count             bigint,
+    compressed_batch_rows_min          bigint,
+    compressed_batch_rows_max          bigint,
+    compressed_batch_rows_sum          bigint,
+    compressed_batch_rows_sqsum        double precision,
+    compressed_batch_bytes_min         bigint,
+    compressed_batch_bytes_max         bigint,
+    compressed_batch_bytes_sum         bigint,
+    compressed_batch_bytes_sqsum       double precision,
+    compressed_block_bytes_min         bigint,
+    compressed_block_bytes_max         bigint,
+    compressed_block_bytes_sum         bigint,
+    compressed_block_bytes_sqsum       double precision,
+    -- DML totals (cumulative, from cmd_totals)
+    total_batches_deleted              bigint,
+    total_batches_decompressed         bigint,
+    total_tuples_decompressed          bigint,
+    total_batches_scanned              bigint,
+    total_batches_checked_by_bloom     bigint,
+    total_batches_pruned_by_bloom      bigint,
+    total_batches_without_bloom        bigint,
+    total_batches_bloom_false_positives bigint,
+    total_batches_filtered_compressed  bigint,
+    total_batches_filtered_decompressed bigint,
+    -- DML last operation snapshot (from cmd_last_op)
+    last_op_batches_deleted            bigint,
+    last_op_batches_decompressed       bigint,
+    last_op_tuples_decompressed        bigint,
+    last_op_batches_scanned            bigint,
+    last_op_batches_checked_by_bloom   bigint,
+    last_op_batches_pruned_by_bloom    bigint,
+    last_op_batches_without_bloom      bigint,
+    last_op_batches_bloom_false_positives bigint,
+    last_op_batches_filtered_compressed bigint,
+    last_op_batches_filtered_decompressed bigint,
+    -- Operation counts
+    n_selects                          bigint,
+    n_inserts                          bigint,
+    n_updates                          bigint,
+    n_deletes                          bigint,
+    -- Timestamps
+    first_update                       timestamptz,
+    last_update                        timestamptz
+)
+AS '@MODULE_PATHNAME@', 'ts_stats_chunks'
+LANGUAGE C STABLE;
+
+-- Logical reset: clears all cached chunks under the segment lock.
+CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_statistics_reset()
+RETURNS VOID
+AS '@MODULE_PATHNAME@', 'ts_stats_reset'
+LANGUAGE C VOLATILE;

sql/updates/reverse-dev.sql

@@ -10,6 +10,11 @@ JOIN pg_constraint con
 WHERE c.osm_chunk
 ON CONFLICT DO NOTHING;
 
+-- Drop chunk stats related objects
+DROP VIEW IF EXISTS timescaledb_information.stat_chunk_activity;
+DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_statistics(regclass, regclass, timestamptz);
+DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_statistics_reset();
+
 -- Restore chunk_constraint rows for outbound FKs by matching chunk-side
 -- FKs to their hypertable-side counterpart by name.
 INSERT INTO _timescaledb_catalog.chunk_constraint

sql/views.sql

@@ -430,5 +430,74 @@ SELECT * FROM timescaledb_information.chunk_compression_settings;
 CREATE OR REPLACE VIEW _timescaledb_config.bgw_job AS
 SELECT * from _timescaledb_catalog.bgw_job;
 
+-- chunk statistics view
+CREATE OR REPLACE VIEW timescaledb_information.stat_chunk_activity AS
+SELECT
+    o.uncompressed_relid::regclass  AS chunk,
+    o.compressed_relid::regclass    AS compressed_chunk,
+    c.id                            AS chunk_id,
+    c.hypertable_id,
+    h.table_name                    AS hypertable,
+    -- Compression
+    o.compressed_batch_count,
+    o.compressed_block_count,
+    o.compressed_batch_rows_min,
+    o.compressed_batch_rows_max,
+    o.compressed_batch_rows_sum / NULLIF(o.compressed_batch_count, 0)
+        AS compressed_batch_rows_avg,
+    sqrt(o.compressed_batch_rows_sqsum / NULLIF(o.compressed_batch_count, 0)
+         - power(o.compressed_batch_rows_sum::double precision
+                 / NULLIF(o.compressed_batch_count, 0), 2))
+        AS compressed_batch_rows_stddev,
+    o.compressed_batch_bytes_min,
+    o.compressed_batch_bytes_max,
+    o.compressed_batch_bytes_sum / NULLIF(o.compressed_batch_count, 0)
+        AS compressed_batch_bytes_avg,
+    sqrt(o.compressed_batch_bytes_sqsum / NULLIF(o.compressed_batch_count, 0)
+         - power(o.compressed_batch_bytes_sum::double precision
+                 / NULLIF(o.compressed_batch_count, 0), 2))
+        AS compressed_batch_bytes_stddev,
+    o.compressed_block_bytes_min,
+    o.compressed_block_bytes_max,
+    o.compressed_block_bytes_sum / NULLIF(o.compressed_block_count, 0)
+        AS compressed_block_bytes_avg,
+    sqrt(o.compressed_block_bytes_sqsum / NULLIF(o.compressed_block_count, 0)
+         - power(o.compressed_block_bytes_sum::double precision
+                 / NULLIF(o.compressed_block_count, 0), 2))
+        AS compressed_block_bytes_stddev,
+    -- CMD totals
+    o.total_batches_deleted,
+    o.total_batches_decompressed,
+    o.total_tuples_decompressed,
+    o.total_batches_scanned,
+    o.total_batches_checked_by_bloom,
+    o.total_batches_pruned_by_bloom,
+    o.total_batches_without_bloom,
+    o.total_batches_bloom_false_positives,
+    o.total_batches_filtered_compressed,
+    o.total_batches_filtered_decompressed,
+    -- last CMD operation
+    o.last_op_batches_deleted,
+    o.last_op_batches_decompressed,
+    o.last_op_tuples_decompressed,
+    o.last_op_batches_scanned,
+    o.last_op_batches_checked_by_bloom,
+    o.last_op_batches_pruned_by_bloom,
+    o.last_op_batches_without_bloom,
+    o.last_op_batches_bloom_false_positives,
+    o.last_op_batches_filtered_compressed,
+    o.last_op_batches_filtered_decompressed,
+    o.n_selects,
+    o.n_inserts,
+    o.n_updates,
+    o.n_deletes,
+    o.first_update,
+    o.last_update
+FROM _timescaledb_functions.chunk_statistics() o
+LEFT JOIN _timescaledb_catalog.chunk c
+       ON format('%I.%I', c.schema_name, c.table_name)::regclass = o.uncompressed_relid
+LEFT JOIN _timescaledb_catalog.hypertable h ON h.id = c.hypertable_id;
+
+
 GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_config TO PUBLIC;
 GRANT SELECT ON ALL TABLES IN SCHEMA timescaledb_information TO PUBLIC;

src/CMakeLists.txt

@@ -125,6 +125,7 @@ add_subdirectory(ts_catalog)
 add_subdirectory(nodes)
 add_subdirectory(planner)
 add_subdirectory(with_clause)
+add_subdirectory(ts_stats)
 
 # Don't run clang-tidy on the files we copied from Postgres. We don't want to
 # introduce changes there unless absolutely necessary. CMake can only access the

src/chunk.c

@@ -40,6 +40,7 @@
 #include <storage/lmgr.h>
 #include <storage/lockdefs.h>
 #include <tcop/tcopprot.h>
+#include <ts_stats/ts_stats_record.h>
 #include <utils/acl.h>
 #include <utils/array.h>
 #include <utils/builtins.h>
@@ -4074,6 +4075,9 @@ ts_chunk_drop(const Chunk *chunk, DropBehavior behavior, int32 log_level)
 
 	/* Drop the table */
 	performDeletion(&objaddr, behavior, 0);
+
+	/* Evict the chunk stats from the shared memory */
+	ts_stats_chunk_evict(chunk->table_id);
 }
 
 static void

src/guc.c

@@ -154,6 +154,8 @@ TSDLLEXPORT bool ts_guc_enable_delete_after_compression = false;
 TSDLLEXPORT bool ts_guc_enable_merge_on_cagg_refresh = false;
 
 bool ts_guc_enable_partitioned_hypertables = false;
+TSDLLEXPORT int ts_guc_stats_max_chunks = TS_STATS_MAX_CHUNKS_DEFAULT;
+
 #if PG16_GE
 TSDLLEXPORT bool ts_guc_enable_cagg_rewrites = false;
 TSDLLEXPORT bool ts_guc_cagg_rewrites_debug_info = false;
@@ -494,6 +496,46 @@ assign_default_chunk_time_interval(const char *newval, void *extra)
 	default_chunk_time_interval = extra;
 }
 
+/*
+ * check_hook: accept 0 (feature disabled) or a power of two in [MIN, MAX].
+ * Non-power-of-two values are rejected with a hint listing valid sizes.
+ */
+static bool
+stats_max_chunks_check_hook(int *newval, void **extra, GucSource source)
+{
+	int v = *newval;
+
+	if (v == 0)
+	{
+		return true; /* 0 = disabled, explicitly allowed */
+	}
+
+	if (v < TS_STATS_MAX_CHUNKS_MIN)
+	{
+		GUC_check_errdetail("Minimum cache capacity is %d chunks. "
+							"Set to 0 to disable the observability feature.",
+							TS_STATS_MAX_CHUNKS_MIN);
+		return false;
+	}
+
+	if (v > TS_STATS_MAX_CHUNKS_MAX)
+	{
+		GUC_check_errdetail("Maximum cache capacity is %d chunks "
+							"(largest power of 2 within the slot index space).",
+							TS_STATS_MAX_CHUNKS_MAX);
+		return false;
+	}
+
+	if ((v & (v - 1)) != 0)
+	{
+		GUC_check_errdetail("timescaledb.stats_max_chunks must be 0 (disabled) or a power of 2.");
+		GUC_check_errhint("Valid values: 0, 256, 512, 1024, 2048, "
+						  "4096, 8192, 16384, 32768, 65536, 131072, 262144.");
+		return false;
+	}
+	return true;
+}
+
 void
 _guc_init(void)
 {
@@ -1443,6 +1485,21 @@ _guc_init(void)
 							 NULL);
 #endif
 
+	DefineCustomIntVariable(MAKE_EXTOPTION("stats_max_chunks"),
+							"Per-database statistics cache capacity, "
+							"in chunks. 0 disables the feature.",
+							"Must be 0 or a power of 2. "
+							"Takes effect only after server restart.",
+							&ts_guc_stats_max_chunks,
+							TS_STATS_MAX_CHUNKS_DEFAULT, /* default */
+							0,							 /* min: 0 = disabled */
+							TS_STATS_MAX_CHUNKS_MAX,	 /* max: 2^18 = 262144 chunks */
+							PGC_SIGHUP,
+							0,
+							stats_max_chunks_check_hook,
+							NULL,
+							NULL);
+
 #ifdef USE_TELEMETRY
 	DefineCustomEnumVariable(MAKE_EXTOPTION("telemetry_level"),
 							 "Telemetry settings level",

src/guc.h

@@ -161,6 +161,13 @@ extern TSDLLEXPORT bool ts_guc_enable_rowlevel_compression_locking;
 extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_batch_sorted_merge;
 
 extern bool ts_guc_enable_partitioned_hypertables;
+extern TSDLLEXPORT int ts_guc_stats_max_chunks;
+
+#define TS_STATS_MAX_CHUNKS_DEFAULT 1024
+#define TS_STATS_MAX_CHUNKS_MIN 256
+#define TS_STATS_MAX_CHUNKS_MAX 262144
+
+#define IS_STATS_CHUNKS_ENABLED() (ts_guc_stats_max_chunks > 0)
 
 void _guc_init(void);
 

src/loader/CMakeLists.txt

@@ -5,6 +5,7 @@ set(SOURCES
     bgw_launcher.c
     bgw_interface.c
     function_telemetry.c
+    ts_stats_handles.c
     lwlocks.c)
 
 set(TEST_SOURCES ${PROJECT_SOURCE_DIR}/test/src/symbol_conflict.c)

src/loader/loader.c

@@ -41,6 +41,7 @@
 #include "loader/function_telemetry.h"
 #include "loader/loader.h"
 #include "loader/lwlocks.h"
+#include "loader/ts_stats_handles.h"
 
 /*
  * Loading process:
@@ -623,6 +624,9 @@ timescaledb_shmem_startup_hook(void)
 	ts_bgw_message_queue_shmem_startup();
 	ts_lwlocks_shmem_startup();
 	ts_function_telemetry_shmem_startup();
+#if PG17_LT
+	ts_stats_shmem_startup();
+#endif
 }
 
 /*
@@ -642,6 +646,9 @@ timescaledb_shmem_request_hook(void)
 	ts_bgw_message_queue_alloc();
 	ts_lwlocks_shmem_alloc();
 	ts_function_telemetry_shmem_alloc();
+#if PG17_LT
+	ts_stats_shmem_request();
+#endif
 }
 
 static void