Use firstlast sparse index for orderby metadata

New compressed chunks now carry both minmax and firstlast
metadata for every orderby column, and the compressed chunk
index is built on the firstlast columns. Predicate pushdown
picks firstlast when the leading orderby is NOT NULL, falling
back to minmax otherwise.

Author: antekresic

.gitignore

@@ -31,6 +31,7 @@ tags
 /compile_commands.json
 /.DS_Store
 /.clangd
+/.claude
 /.cache
 
 /CMakeSettings.json

.unreleased/pr_9784

@@ -0,0 +1 @@
+Implements: #9784 Use firstlast sparse index for orderby metadata on new compressed chunks

src/ts_catalog/compression_settings.c

@@ -964,27 +964,41 @@ ts_add_orderby_sparse_index(CompressionSettings *settings)
 		}
 	}
 
-	/* add orderby sparse settings */
+	/*
+	 * For each orderby column, record both the minmax and firstlast sparse
+	 * indexes that the compressed chunk carries by default. The physical
+	 * columns are always created for orderby columns, so the index
+	 * configuration should reflect that.
+	 *
+	 * If the column already appears anywhere in the user-supplied index
+	 * configuration, leave it alone so user intent wins.
+	 */
 	ArrayIterator it = array_create_iterator(settings->fd.orderby, 0, NULL);
 	while (array_iterate(it, &datum, &isnull))
 	{
-		/*
-		 * check if sparse index for column already exists
-		 * Validation is done by ts_compression_settings_update
-		 */
+		char *col_name = TextDatumGetCString(datum);
+
 		if (settings->fd.index &&
 			ts_jsonb_has_key_value_str_field(settings->fd.index,
 											 ts_sparse_index_common_keys[SparseIndexKeyCol],
-											 TextDatumGetCString(datum)))
+											 col_name))
 		{
 			continue;
 		}
 
-		MinmaxIndexColumnConfig config;
-		config.base.type = _SparseIndexTypeEnumMinmax;
-		config.col = TextDatumGetCString(datum);
-		config.base.source = _SparseIndexSourceEnumOrderby;
-		ts_convert_sparse_index_config_to_jsonb(parse_state, (SparseIndexConfigBase *) &config);
+		MinmaxIndexColumnConfig minmax_config;
+		minmax_config.base.type = _SparseIndexTypeEnumMinmax;
+		minmax_config.base.source = _SparseIndexSourceEnumOrderby;
+		minmax_config.col = col_name;
+		ts_convert_sparse_index_config_to_jsonb(parse_state,
+												(SparseIndexConfigBase *) &minmax_config);
+
+		FirstLastIndexColumnConfig firstlast_config;
+		firstlast_config.base.type = _SparseIndexTypeEnumFirstLast;
+		firstlast_config.base.source = _SparseIndexSourceEnumOrderby;
+		firstlast_config.col = col_name;
+		ts_convert_sparse_index_config_to_jsonb(parse_state,
+												(SparseIndexConfigBase *) &firstlast_config);
 	}
 
 	return JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_ARRAY, NULL));

tsl/src/chunk_split.c

@@ -1210,22 +1210,27 @@ chunk_split_chunk(PG_FUNCTION_ARGS)
 			   NameStr(splitdim_name));
 
 		/*
-		 * Get the attribute numbers for the primary dimension's min and max
-		 * values in the compressed relation. We'll use these to get the time
-		 * range of compressed segments in order to route segments to the
-		 * right result chunk.
+		 * Get the attribute numbers for the primary dimension's lower- and
+		 * upper-bound metadata columns in the compressed relation. We'll use
+		 * these to get the time range of compressed segments in order to
+		 * route segments to the right result chunk.
 		 */
-		const char *min_attname = column_segment_min_name(orderby_pos);
-		const char *max_attname = column_segment_max_name(orderby_pos);
+		AttrNumber lower_attno;
+		AttrNumber upper_attno;
+		orderby_sparse_metadata_attnos(compress_settings,
+									   compress_settings->fd.compress_relid,
+									   orderby_pos,
+									   &lower_attno,
+									   &upper_attno);
 
 		CompressedSplitPoint csp = {
 			.base = {
 				.point = split_at,
 				.dim = hyperspace_get_open_dimension(ht->space, 0),
 				.route_next_tuple = route_next_compressed_tuple,
 			},
-			.attnum_min = get_attnum(compress_settings->fd.compress_relid, min_attname),
-			.attnum_max = get_attnum(compress_settings->fd.compress_relid, max_attname),
+			.attnum_min = lower_attno,
+			.attnum_max = upper_attno,
 			.attnum_count = get_attnum(compress_settings->fd.compress_relid, COMPRESSION_COLUMN_METADATA_COUNT_NAME),
 			.noncompressed_tupdesc = CreateTupleDescCopy(RelationGetDescr(srcrel)),
 		};

tsl/src/compression/compression.c

@@ -942,9 +942,6 @@ build_column_map(const CompressionSettings *settings, const TupleDesc in_desc,
 																	 segment_max_attr_offset));
 				}
 
-				Ensure(!is_orderby || has_minmax_metadata,
-					   "orderby columns must have minmax metadata");
-
 				const AttrNumber bloom_attr_number =
 					compressed_column_metadata_attno(settings,
 													 settings->fd.relid,
@@ -976,9 +973,11 @@ build_column_map(const CompressionSettings *settings, const TupleDesc in_desc,
 													 attr->attnum,
 													 settings->fd.compress_relid,
 													 "last");
+				bool has_firstlast_metadata = false;
 				if (AttributeNumberIsValid(first_attr_number) &&
 					AttributeNumberIsValid(last_attr_number))
 				{
+					has_firstlast_metadata = true;
 					const int16 first_attr_offset = AttrNumberGetAttrOffset(first_attr_number);
 					const int16 last_attr_offset = AttrNumberGetAttrOffset(last_attr_number);
 					metadata_builders =
@@ -989,6 +988,9 @@ build_column_map(const CompressionSettings *settings, const TupleDesc in_desc,
 																		last_attr_offset));
 				}
 
+				Ensure(!is_orderby || has_minmax_metadata || has_firstlast_metadata,
+					   "orderby columns must have sparse index metadata");
+
 				*column = (PerColumn){
 					.compressor = compressor_for_type(attr->atttypid),
 					.segmentby_column_index = -1,

tsl/src/compression/compression_storage.c

@@ -331,10 +331,14 @@ create_compressed_chunk_indexes(Chunk *chunk, CompressionSettings *settings)
 	initStringInfo(&orderby_buf);
 	for (int i = 1; i <= ts_array_length(settings->fd.orderby); i++)
 	{
+		char *lower_name;
+		char *upper_name;
+		orderby_sparse_metadata_names(settings, i, &lower_name, &upper_name);
+
 		resetStringInfo(&orderby_buf);
-		/* Add min metadata column */
-		IndexElem *orderby_min_elem = makeNode(IndexElem);
-		orderby_min_elem->name = column_segment_min_name(i);
+		/* Lower-boundary metadata column. */
+		IndexElem *orderby_lower_elem = makeNode(IndexElem);
+		orderby_lower_elem->name = lower_name;
 		if (ts_array_get_element_bool(settings->fd.orderby_desc, i))
 		{
 			appendStringInfoString(&orderby_buf, " DESC");
@@ -345,11 +349,11 @@ create_compressed_chunk_indexes(Chunk *chunk, CompressionSettings *settings)
 			appendStringInfoString(&orderby_buf, " ASC");
 			ordering = SORTBY_ASC;
 		}
-		orderby_min_elem->ordering = ordering;
+		orderby_lower_elem->ordering = ordering;
 
 		if (ts_array_get_element_bool(settings->fd.orderby_nullsfirst, i))
 		{
-			if (orderby_min_elem->ordering != SORTBY_DESC)
+			if (orderby_lower_elem->ordering != SORTBY_DESC)
 			{
 				appendStringInfoString(&orderby_buf, " NULLS FIRST");
 				nulls_ordering = SORTBY_NULLS_FIRST;
@@ -361,7 +365,7 @@ create_compressed_chunk_indexes(Chunk *chunk, CompressionSettings *settings)
 		}
 		else
 		{
-			if (orderby_min_elem->ordering != SORTBY_DESC)
+			if (orderby_lower_elem->ordering != SORTBY_DESC)
 			{
 				nulls_ordering = SORTBY_NULLS_DEFAULT;
 			}
@@ -371,21 +375,21 @@ create_compressed_chunk_indexes(Chunk *chunk, CompressionSettings *settings)
 				nulls_ordering = SORTBY_NULLS_LAST;
 			}
 		}
-		orderby_min_elem->nulls_ordering = nulls_ordering;
-		appendStringInfoString(&buf, orderby_min_elem->name);
+		orderby_lower_elem->nulls_ordering = nulls_ordering;
+		appendStringInfoString(&buf, orderby_lower_elem->name);
 		appendStringInfoString(&buf, orderby_buf.data);
 		appendStringInfoString(&buf, ", ");
-		indexcols = lappend(indexcols, orderby_min_elem);
-
-		/* Add max metadata column */
-		IndexElem *orderby_max_elem = makeNode(IndexElem);
-		orderby_max_elem->name = column_segment_max_name(i);
-		orderby_max_elem->ordering = orderby_min_elem->ordering;
-		orderby_max_elem->nulls_ordering = orderby_min_elem->nulls_ordering;
-		appendStringInfoString(&buf, orderby_max_elem->name);
+		indexcols = lappend(indexcols, orderby_lower_elem);
+
+		/* Upper-boundary metadata column. */
+		IndexElem *orderby_upper_elem = makeNode(IndexElem);
+		orderby_upper_elem->name = upper_name;
+		orderby_upper_elem->ordering = orderby_lower_elem->ordering;
+		orderby_upper_elem->nulls_ordering = orderby_lower_elem->nulls_ordering;
+		appendStringInfoString(&buf, orderby_upper_elem->name);
 		appendStringInfoString(&buf, orderby_buf.data);
 		appendStringInfoString(&buf, ", ");
-		indexcols = lappend(indexcols, orderby_max_elem);
+		indexcols = lappend(indexcols, orderby_upper_elem);
 	}
 
 	stmt.indexParams = indexcols;

tsl/src/compression/create.c

@@ -156,6 +156,79 @@ column_segment_max_name(int16 column_index)
 	return compression_column_segment_metadata_name("max", column_index);
 }
 
+OrderbySparseKind
+orderby_sparse_kind(const CompressionSettings *settings, int orderby_pos)
+{
+	Assert(settings != NULL);
+	Assert(orderby_pos >= 1 && orderby_pos <= ts_array_length(settings->fd.orderby));
+
+	if (settings->fd.index == NULL)
+	{
+		return ORDERBY_SPARSE_MINMAX;
+	}
+
+	const char *col_name = ts_array_get_element_text(settings->fd.orderby, orderby_pos);
+
+	SparseIndexSettings *parsed = ts_convert_to_sparse_index_settings(settings->fd.index);
+	List *per_col = ts_get_per_column_compression_settings(parsed);
+	PerColumnCompressionSettings *col_settings =
+		ts_get_per_column_compression_settings_by_column_name(per_col, col_name);
+
+	OrderbySparseKind kind = ORDERBY_SPARSE_MINMAX;
+	if (col_settings != NULL && col_settings->firstlast_obj_id >= 0)
+	{
+		kind = ORDERBY_SPARSE_FIRSTLAST;
+	}
+
+	ts_free_sparse_index_settings(parsed);
+	return kind;
+}
+
+void
+orderby_sparse_metadata_names(const CompressionSettings *settings, int orderby_pos,
+							  char **lower_name, char **upper_name)
+{
+	Assert(lower_name != NULL && upper_name != NULL);
+
+	OrderbySparseKind kind = orderby_sparse_kind(settings, orderby_pos);
+
+	if (kind == ORDERBY_SPARSE_MINMAX)
+	{
+		*lower_name = column_segment_min_name(orderby_pos);
+		*upper_name = column_segment_max_name(orderby_pos);
+		return;
+	}
+
+	const char *col_name = ts_array_get_element_text(settings->fd.orderby, orderby_pos);
+	const char *col_names[1] = { col_name };
+	bool desc = ts_array_get_element_bool(settings->fd.orderby_desc, orderby_pos);
+
+	if (!desc)
+	{
+		*lower_name = compressed_column_metadata_name_v2("first", col_names, 1);
+		*upper_name = compressed_column_metadata_name_v2("last", col_names, 1);
+	}
+	else
+	{
+		/* DESC sort puts the largest value at the first row of each batch. */
+		*lower_name = compressed_column_metadata_name_v2("last", col_names, 1);
+		*upper_name = compressed_column_metadata_name_v2("first", col_names, 1);
+	}
+}
+
+void
+orderby_sparse_metadata_attnos(const CompressionSettings *settings, Oid compressed_relid,
+							   int orderby_pos, AttrNumber *lower_attno, AttrNumber *upper_attno)
+{
+	Assert(lower_attno != NULL && upper_attno != NULL);
+
+	char *lower_name;
+	char *upper_name;
+	orderby_sparse_metadata_names(settings, orderby_pos, &lower_name, &upper_name);
+	*lower_attno = get_attnum(compressed_relid, lower_name);
+	*upper_attno = get_attnum(compressed_relid, upper_name);
+}
+
 /*
  * Get metadata name for a given column name and metadata type, format version 2.
  * We can't reference the attribute numbers, because they can change after
@@ -582,7 +655,11 @@ build_columndefs(CompressionSettings *settings, Oid src_reloid)
 						 errdetail("Could not identify a less-than operator for the type.")));
 			}
 
-			/* segment_meta min and max columns */
+			/*
+			 * Every orderby column gets minmax metadata so range predicates
+			 * on any orderby position can be pushed down as a filter on the
+			 * compressed scan.
+			 */
 			ColumnDef *def = makeColumnDef(column_segment_min_name(index),
 										   attr->atttypid,
 										   attr->atttypmod,
@@ -595,6 +672,29 @@ build_columndefs(CompressionSettings *settings, Oid src_reloid)
 								attr->attcollation);
 			def->storage = TYPSTORAGE_PLAIN;
 			compressed_column_defs = lappend(compressed_column_defs, def);
+
+			/*
+			 * When firstlast is configured for the orderby column, also add
+			 * first/last metadata. These columns are a part of the compressed chunk
+			 * btree and let predicates on the leading orderby become index
+			 * conditions when the column is NOT NULL.
+			 */
+			bool add_firstlast = per_column_setting != NULL && composite_attr_lists != NULL &&
+								 per_column_setting->firstlast_obj_id != -1;
+			if (add_firstlast)
+			{
+				ColumnDef *first_def =
+					create_sparse_index_column_def(composite_attr_lists[per_column_setting
+																			->firstlast_obj_id],
+												   "first");
+				compressed_column_defs = lappend(compressed_column_defs, first_def);
+
+				ColumnDef *last_def =
+					create_sparse_index_column_def(composite_attr_lists[per_column_setting
+																			->firstlast_obj_id],
+												   "last");
+				compressed_column_defs = lappend(compressed_column_defs, last_def);
+			}
 		}
 		else if (per_column_setting != NULL && composite_attr_lists != NULL)
 		{
@@ -604,12 +704,11 @@ build_columndefs(CompressionSettings *settings, Oid src_reloid)
 			bool is_firstlast = per_column_setting->firstlast_obj_id != -1;
 
 			/*
-			 * We allow only one sparse index per column. Columns used in the ORDER BY
-			 * clause implicitly have a minmax index and adding a bloom filter on them is not
-			 * allowed.
+			 * We allow only one sparse index per column. Adding a bloom filter
+			 * on a column that already has a sparse index is not allowed.
 			 *
-			 * The parser is expected to enforce this constraint earlier, but we check again
-			 * here as a safeguard.
+			 * The parser is expected to enforce this constraint earlier, but we
+			 * check again here as a safeguard.
 			 */
 			Ensure((!is_bloom || !is_minmax),
 				   "Should not create bloom filter for minmax column \"%s\"",

tsl/src/compression/create.h

@@ -37,6 +37,33 @@ int compressed_column_metadata_attno(const CompressionSettings *settings, Oid ch
 									 AttrNumber chunk_attno, Oid compressed_reloid,
 									 char const *metadata_type);
 
+/*
+ * Per-orderby-column sparse-index metadata dispatch.
+ *
+ * Each orderby has a pair of metadata columns that bound the orderby values
+ * in each batch. Two shapes are supported:
+ *
+ *   minmax    - statistical extremes; direction-blind (lower=min, upper=max).
+ *   firstlast - first/last row values in sort order. Under ASC, lower=first
+ *               and upper=last; under DESC, the roles flip.
+ *
+ * Callers should ask for "lower"/"upper" boundaries instead of min/max
+ * directly so the same code works across both shapes. orderby_pos is 1-based
+ * to match settings->fd.orderby and the column_segment_*_name helpers.
+ */
+typedef enum OrderbySparseKind
+{
+	ORDERBY_SPARSE_MINMAX,
+	ORDERBY_SPARSE_FIRSTLAST,
+} OrderbySparseKind;
+
+OrderbySparseKind orderby_sparse_kind(const CompressionSettings *settings, int orderby_pos);
+void orderby_sparse_metadata_names(const CompressionSettings *settings, int orderby_pos,
+								   char **lower_name, char **upper_name);
+void orderby_sparse_metadata_attnos(const CompressionSettings *settings, Oid compressed_relid,
+									int orderby_pos, AttrNumber *lower_attno,
+									AttrNumber *upper_attno);
+
 void tsl_columnstore_setup(Hypertable *ht, WithClauseResult *with_clause_options);
 
 void compression_settings_set_defaults(Hypertable *ht, CompressionSettings *settings,