Observability for chunk data

This change collects information about compressed chunks and stores
them in an LRU fashion. It captures events at compression, query
and DML oparations. The query/DML operations maintain totals and
information about the last operation.

The chunk information is maintained in shared memory per database.
One can access the data via calling the

 `_timescaledb_functions.observ_chunks(..)` function or by

selecting from the

 `timescaledb_information.observ_chunks` view

the former allows the user to push down filters, so it is faster
and less lock contention.

The amount of shared memory is controlled by the GUC:

 `timescaledb.observ_max_chunks`

If the GUC is set to zero, the feature is disabled. The default
value is 1024 and the maximum is 32768. The amount must be a
power of two number. The `observ_max_chunks` value controls
how many chunk we store information about. The memory consumption
is 300 bytes per entry.

Author: dbeck

.unreleased/pr_9730

@@ -0,0 +1 @@
+Implements: #9730 In memory observability for compressed chunks

cmake/ScriptFiles.cmake

@@ -42,6 +42,7 @@ set(SOURCE_FILES
     bgw_scheduler.sql
     metadata.sql
     uuidv7.sql
+    ts_stats.sql
     views.sql
     views_experimental.sql
     gapfill.sql

sql/ts_stats.sql

@@ -0,0 +1,71 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+-- Chunk statistics. This function takes three optional parameters:
+-- - compressed_relid: if provided, returns stats for the specified compressed chunk only; otherwise, returns stats for all chunks.
+-- - uncompressed_relid: if provided, returns stats for the specified user chunk only; if compressed_relid is also provided, both must agree.
+-- - since: if provided, returns stats for operations that occurred since the specified timestamp; otherwise, returns all stats.
+--
+CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_statistics(
+    compressed_relid   regclass    DEFAULT NULL,
+    uncompressed_relid regclass    DEFAULT NULL,
+    since              timestamptz DEFAULT NULL
+)
+RETURNS TABLE (
+    compressed_relid                   oid,
+    uncompressed_relid                 oid,
+    -- Compression
+    compressed_batch_count             bigint,
+    compressed_block_count             bigint,
+    compressed_batch_rows_min          bigint,
+    compressed_batch_rows_max          bigint,
+    compressed_batch_rows_sum          bigint,
+    compressed_batch_rows_sqsum        double precision,
+    compressed_batch_bytes_min         bigint,
+    compressed_batch_bytes_max         bigint,
+    compressed_batch_bytes_sum         bigint,
+    compressed_batch_bytes_sqsum       double precision,
+    compressed_block_bytes_min         bigint,
+    compressed_block_bytes_max         bigint,
+    compressed_block_bytes_sum         bigint,
+    compressed_block_bytes_sqsum       double precision,
+    -- DML totals (cumulative, from cmd_totals)
+    total_batches_deleted              bigint,
+    total_batches_decompressed         bigint,
+    total_tuples_decompressed          bigint,
+    total_batches_scanned              bigint,
+    total_batches_checked_by_bloom     bigint,
+    total_batches_pruned_by_bloom      bigint,
+    total_batches_without_bloom        bigint,
+    total_batches_bloom_false_positives bigint,
+    total_batches_filtered_compressed  bigint,
+    total_batches_filtered_decompressed bigint,
+    -- DML last operation snapshot (from cmd_last_op)
+    last_op_batches_deleted            bigint,
+    last_op_batches_decompressed       bigint,
+    last_op_tuples_decompressed        bigint,
+    last_op_batches_scanned            bigint,
+    last_op_batches_checked_by_bloom   bigint,
+    last_op_batches_pruned_by_bloom    bigint,
+    last_op_batches_without_bloom      bigint,
+    last_op_batches_bloom_false_positives bigint,
+    last_op_batches_filtered_compressed bigint,
+    last_op_batches_filtered_decompressed bigint,
+    -- Operation counts
+    n_selects                          bigint,
+    n_inserts                          bigint,
+    n_updates                          bigint,
+    n_deletes                          bigint,
+    -- Timestamps
+    first_update                       timestamptz,
+    last_update                        timestamptz
+)
+AS '@MODULE_PATHNAME@', 'ts_stats_chunks'
+LANGUAGE C STABLE;
+
+-- Logical reset: clears all cached chunks under the segment lock.
+CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_statistics_reset()
+RETURNS VOID
+AS '@MODULE_PATHNAME@', 'ts_stats_reset'
+LANGUAGE C VOLATILE;

sql/updates/reverse-dev.sql

@@ -9,3 +9,8 @@ DROP FUNCTION IF EXISTS _timescaledb_functions.bloom1_hash(anyelement);
 -- Drop BIGINT-returning version so the downgrade script can recreate the
 -- INTEGER-returning version of compressed_data_column_size.
 DROP FUNCTION IF EXISTS _timescaledb_functions.compressed_data_column_size(_timescaledb_internal.compressed_data, ANYELEMENT);
+
+DROP VIEW IF EXISTS timescaledb_information.stat_chunk_activity;
+DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_statistics(regclass, regclass, timestamptz);
+DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_statistics_reset();
+

sql/views.sql

@@ -431,5 +431,74 @@ SELECT * FROM timescaledb_information.chunk_compression_settings;
 CREATE OR REPLACE VIEW _timescaledb_config.bgw_job AS
 SELECT * from _timescaledb_catalog.bgw_job;
 
+-- chunk statistics view
+CREATE OR REPLACE VIEW timescaledb_information.stat_chunk_activity AS
+SELECT
+    o.uncompressed_relid::regclass  AS chunk,
+    o.compressed_relid::regclass    AS compressed_chunk,
+    c.id                            AS chunk_id,
+    c.hypertable_id,
+    h.table_name                    AS hypertable,
+    -- Compression
+    o.compressed_batch_count,
+    o.compressed_block_count,
+    o.compressed_batch_rows_min,
+    o.compressed_batch_rows_max,
+    o.compressed_batch_rows_sum / NULLIF(o.compressed_batch_count, 0)
+        AS compressed_batch_rows_avg,
+    sqrt(o.compressed_batch_rows_sqsum / NULLIF(o.compressed_batch_count, 0)
+         - power(o.compressed_batch_rows_sum::double precision
+                 / NULLIF(o.compressed_batch_count, 0), 2))
+        AS compressed_batch_rows_stddev,
+    o.compressed_batch_bytes_min,
+    o.compressed_batch_bytes_max,
+    o.compressed_batch_bytes_sum / NULLIF(o.compressed_batch_count, 0)
+        AS compressed_batch_bytes_avg,
+    sqrt(o.compressed_batch_bytes_sqsum / NULLIF(o.compressed_batch_count, 0)
+         - power(o.compressed_batch_bytes_sum::double precision
+                 / NULLIF(o.compressed_batch_count, 0), 2))
+        AS compressed_batch_bytes_stddev,
+    o.compressed_block_bytes_min,
+    o.compressed_block_bytes_max,
+    o.compressed_block_bytes_sum / NULLIF(o.compressed_block_count, 0)
+        AS compressed_block_bytes_avg,
+    sqrt(o.compressed_block_bytes_sqsum / NULLIF(o.compressed_block_count, 0)
+         - power(o.compressed_block_bytes_sum::double precision
+                 / NULLIF(o.compressed_block_count, 0), 2))
+        AS compressed_block_bytes_stddev,
+    -- CMD totals
+    o.total_batches_deleted,
+    o.total_batches_decompressed,
+    o.total_tuples_decompressed,
+    o.total_batches_scanned,
+    o.total_batches_checked_by_bloom,
+    o.total_batches_pruned_by_bloom,
+    o.total_batches_without_bloom,
+    o.total_batches_bloom_false_positives,
+    o.total_batches_filtered_compressed,
+    o.total_batches_filtered_decompressed,
+    -- last CMD operation
+    o.last_op_batches_deleted,
+    o.last_op_batches_decompressed,
+    o.last_op_tuples_decompressed,
+    o.last_op_batches_scanned,
+    o.last_op_batches_checked_by_bloom,
+    o.last_op_batches_pruned_by_bloom,
+    o.last_op_batches_without_bloom,
+    o.last_op_batches_bloom_false_positives,
+    o.last_op_batches_filtered_compressed,
+    o.last_op_batches_filtered_decompressed,
+    o.n_selects,
+    o.n_inserts,
+    o.n_updates,
+    o.n_deletes,
+    o.first_update,
+    o.last_update
+FROM _timescaledb_functions.chunk_statistics() o
+LEFT JOIN _timescaledb_catalog.chunk c
+       ON format('%I.%I', c.schema_name, c.table_name)::regclass = o.uncompressed_relid
+LEFT JOIN _timescaledb_catalog.hypertable h ON h.id = c.hypertable_id;
+
+
 GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_config TO PUBLIC;
 GRANT SELECT ON ALL TABLES IN SCHEMA timescaledb_information TO PUBLIC;

src/CMakeLists.txt

@@ -126,6 +126,7 @@ add_subdirectory(ts_catalog)
 add_subdirectory(nodes)
 add_subdirectory(planner)
 add_subdirectory(with_clause)
+add_subdirectory(ts_stats)
 
 # Don't run clang-tidy on the files we copied from Postgres. We don't want to
 # introduce changes there unless absolutely necessary. CMake can only access the

src/guc.c

@@ -153,6 +153,8 @@ TSDLLEXPORT bool ts_guc_enable_delete_after_compression = false;
 TSDLLEXPORT bool ts_guc_enable_merge_on_cagg_refresh = false;
 
 bool ts_guc_enable_partitioned_hypertables = false;
+TSDLLEXPORT int ts_guc_observ_max_chunks = TS_OBSERV_MAX_CHUNKS_DEFAULT;
+
 #if PG16_GE
 TSDLLEXPORT bool ts_guc_enable_cagg_rewrites = false;
 TSDLLEXPORT bool ts_guc_cagg_rewrites_debug_info = false;
@@ -493,6 +495,46 @@ assign_default_chunk_time_interval(const char *newval, void *extra)
 	default_chunk_time_interval = extra;
 }
 
+/*
+ * check_hook: accept 0 (feature disabled) or a power of two in [MIN, MAX].
+ * Non-power-of-two values are rejected with a hint listing valid sizes.
+ */
+static bool
+observ_max_chunks_check_hook(int *newval, void **extra, GucSource source)
+{
+	int v = *newval;
+
+	if (v == 0)
+	{
+		return true; /* 0 = disabled, explicitly allowed */
+	}
+
+	if (v < TS_OBSERV_MAX_CHUNKS_MIN)
+	{
+		GUC_check_errdetail("Minimum cache capacity is %d chunks. "
+							"Set to 0 to disable the observability feature.",
+							TS_OBSERV_MAX_CHUNKS_MIN);
+		return false;
+	}
+
+	if (v > TS_OBSERV_MAX_CHUNKS_MAX)
+	{
+		GUC_check_errdetail("Maximum cache capacity is %d chunks "
+							"(largest power of 2 within the slot index space).",
+							TS_OBSERV_MAX_CHUNKS_MAX);
+		return false;
+	}
+
+	if ((v & (v - 1)) != 0)
+	{
+		GUC_check_errdetail("timescaledb.observ_max_chunks must be 0 (disabled) or a power of 2.");
+		GUC_check_errhint("Valid values: 0, 64, 128, 256, 512, 1024, 2048, "
+						  "4096, 8192, 16384, 32768.");
+		return false;
+	}
+	return true;
+}
+
 void
 _guc_init(void)
 {
@@ -1423,6 +1465,22 @@ _guc_init(void)
 							 NULL);
 #endif
 
+	DefineCustomIntVariable(MAKE_EXTOPTION("observ_max_chunks"),
+							"Per-database observability cache capacity, "
+							"in chunks. 0 disables the feature.",
+							"Must be 0 or a power of 2. "
+							"Takes effect only for databases whose observability segment "
+							"has not yet been created.",
+							&ts_guc_observ_max_chunks,
+							TS_OBSERV_MAX_CHUNKS_DEFAULT, /* default */
+							0,							  /* min: 0 = disabled */
+							TS_OBSERV_MAX_CHUNKS_MAX,	  /* max: 2^15 = 32768 chunks */
+							PGC_SIGHUP,
+							0,
+							observ_max_chunks_check_hook,
+							NULL,
+							NULL);
+
 #ifdef USE_TELEMETRY
 	DefineCustomEnumVariable(MAKE_EXTOPTION("telemetry_level"),
 							 "Telemetry settings level",

src/guc.h

@@ -160,6 +160,13 @@ extern TSDLLEXPORT bool ts_guc_enable_rowlevel_compression_locking;
 extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_batch_sorted_merge;
 
 extern bool ts_guc_enable_partitioned_hypertables;
+extern TSDLLEXPORT int ts_guc_observ_max_chunks;
+
+#define TS_OBSERV_MAX_CHUNKS_DEFAULT 1024
+#define TS_OBSERV_MAX_CHUNKS_MIN 64
+#define TS_OBSERV_MAX_CHUNKS_MAX 32768 /* largest power of two <= INVALID_IDX */
+
+#define IS_OBSERV_CHUNKS_ENABLED() (ts_guc_observ_max_chunks > 0)
 
 void _guc_init(void);
 

src/loader/CMakeLists.txt

@@ -5,6 +5,7 @@ set(SOURCES
     bgw_launcher.c
     bgw_interface.c
     function_telemetry.c
+    ts_stats_handles.c
     lwlocks.c)
 
 set(TEST_SOURCES ${PROJECT_SOURCE_DIR}/test/src/symbol_conflict.c)

src/loader/loader.c

@@ -41,6 +41,7 @@
 #include "loader/function_telemetry.h"
 #include "loader/loader.h"
 #include "loader/lwlocks.h"
+#include "loader/ts_stats_handles.h"
 
 /*
  * Loading process:
@@ -623,6 +624,9 @@ timescaledb_shmem_startup_hook(void)
 	ts_bgw_message_queue_shmem_startup();
 	ts_lwlocks_shmem_startup();
 	ts_function_telemetry_shmem_startup();
+#if PG17_LT
+	ts_stats_shmem_startup();
+#endif
 }
 
 /*
@@ -642,6 +646,9 @@ timescaledb_shmem_request_hook(void)
 	ts_bgw_message_queue_alloc();
 	ts_lwlocks_shmem_alloc();
 	ts_function_telemetry_shmem_alloc();
+#if PG17_LT
+	ts_stats_shmem_request();
+#endif
 }
 
 static void