Skip to content

Commit 636e04c

Browse files
antekresicantekresic
committed
Add firstlast sparse index type for compression
Add a new sparse index type that stores the actual first and last values of each compressed batch in sort order, including NULLs. Unlike minmax, which skips NULLs and tracks statistical extremes, firstlast tracks positional boundary values. This enables tracking batch ordering and NULL presence at batch boundaries.
1 parent a3f0d75 commit 636e04c

13 files changed

Lines changed: 860 additions & 13 deletions

‎.unreleased/pr_9580‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Implements: #9504 Add first/last sparse indexes to compression

‎src/ts_catalog/compression_settings.c‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
#include <common/md5.h>
1919
#include <utils/palloc.h>
2020

21-
TSDLLEXPORT const char *ts_sparse_index_type_names[] = { "bloom", "minmax" };
21+
TSDLLEXPORT const char *ts_sparse_index_type_names[] = { "bloom", "minmax", "firstlast" };
2222
TSDLLEXPORT const char *ts_sparse_index_source_names[] = { "config", "default", "orderby" };
2323
TSDLLEXPORT const char *ts_sparse_index_common_keys[] = { "type", "column", "source", NULL };
2424
static ScanTupleResult compression_settings_tuple_update(TupleInfo *ti, void *data);
@@ -560,6 +560,14 @@ ts_convert_sparse_index_config_to_jsonb(JsonbParseState *parse_state, SparseInde
560560
ts_sparse_index_common_keys[SparseIndexKeyCol],
561561
minmax_config->col); /* column */
562562
break;
563+
case _SparseIndexTypeEnumFirstLast:
564+
{
565+
FirstLastIndexColumnConfig *firstlast_config = (FirstLastIndexColumnConfig *) config;
566+
ts_jsonb_add_str(parse_state,
567+
ts_sparse_index_common_keys[SparseIndexKeyCol],
568+
firstlast_config->col); /* column */
569+
break;
570+
}
563571
case _SparseIndexTypeEnumBloom:
564572
bloom_config = (BloomFilterConfig *) config;
565573

@@ -1230,6 +1238,7 @@ ts_get_per_column_compression_settings(const SparseIndexSettings *settings)
12301238
per_column_setting = palloc0(sizeof(PerColumnCompressionSettings));
12311239
per_column_setting->column_name = column_name;
12321240
per_column_setting->minmax_obj_id = -1;
1241+
per_column_setting->firstlast_obj_id = -1;
12331242
per_column_setting->single_bloom_obj_id = -1;
12341243
per_column_setting->composite_bloom_index_obj_ids = NULL;
12351244
result_settings = lappend(result_settings, per_column_setting);
@@ -1240,6 +1249,12 @@ ts_get_per_column_compression_settings(const SparseIndexSettings *settings)
12401249
Assert(num_columns == 1);
12411250
per_column_setting->minmax_obj_id = obj_id;
12421251
}
1252+
else if (strcmp(index_type,
1253+
ts_sparse_index_type_names[_SparseIndexTypeEnumFirstLast]) == 0)
1254+
{
1255+
Assert(num_columns == 1);
1256+
per_column_setting->firstlast_obj_id = obj_id;
1257+
}
12431258
else if (strcmp(index_type,
12441259
ts_sparse_index_type_names[_SparseIndexTypeEnumBloom]) == 0)
12451260
{

‎src/ts_catalog/compression_settings.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ typedef enum SparseIndexTypeEnum
2020
{
2121
_SparseIndexTypeEnumBloom = 0,
2222
_SparseIndexTypeEnumMinmax,
23+
_SparseIndexTypeEnumFirstLast,
2324
_SparseIndexTypeEnumMax
2425
} SparseIndexTypeEnum;
2526

@@ -55,6 +56,12 @@ typedef struct MinmaxIndexColumnConfig
5556
const char *col;
5657
} MinmaxIndexColumnConfig;
5758

59+
typedef struct FirstLastIndexColumnConfig
60+
{
61+
SparseIndexConfigBase base;
62+
const char *col;
63+
} FirstLastIndexColumnConfig;
64+
5865
typedef struct SparseIndexColumn
5966
{
6067
/* composite bloom indexes will have multiple SparseIndexColumn entries and
@@ -116,6 +123,9 @@ typedef struct PerColumnCompressionSettings
116123
/* the index of the minmax index object that the column participates in, -1 if not present */
117124
int minmax_obj_id;
118125

126+
/* the index of the firstlast index object that the column participates in, -1 if not present */
127+
int firstlast_obj_id;
128+
119129
/* the index of the single bloom index object that the column participates in, -1 if not present
120130
*/
121131
int single_bloom_obj_id;

‎src/with_clause/alter_table_with_clause.c‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ static const WithClauseDefinition sparse_index_with_clause_def[] = {
6868
.arg_names = {"compress_minmax", "minmax", "compress_min_max", "min_max", NULL},
6969
.type_id = TEXTOID,
7070
},
71+
[_SparseIndexTypeEnumFirstLast] = {
72+
.arg_names = {"compress_firstlast", "firstlast", "first_last", NULL},
73+
.type_id = TEXTOID,
74+
},
7175
};
7276

7377
WithClauseResult *
@@ -477,6 +481,7 @@ parse_sparse_index_config(JsonbParseState *parse_state, FuncCall *sparse_index_d
477481
{
478482
TypeCacheEntry *type_cache;
479483
MinmaxIndexColumnConfig minmax_config;
484+
FirstLastIndexColumnConfig firstlast_config;
480485
BloomFilterConfig bloom_config;
481486
SparseIndexConfigBase config;
482487
SparseIndexConfigBase *config_ptr = &config;
@@ -511,7 +516,7 @@ parse_sparse_index_config(JsonbParseState *parse_state, FuncCall *sparse_index_d
511516
{
512517
ereport(ERROR,
513518
(errcode(ERRCODE_SYNTAX_ERROR),
514-
errmsg("minmax index can only have one column")));
519+
errmsg("only bloom indexes can have multiple columns")));
515520
}
516521
}
517522

@@ -614,6 +619,22 @@ parse_sparse_index_config(JsonbParseState *parse_state, FuncCall *sparse_index_d
614619
minmax_config.col = first_column.name;
615620
break;
616621

622+
case _SparseIndexTypeEnumFirstLast:
623+
if (ts_bmslist_contains_set(*sparse_index_columns, attnums_bitmap))
624+
{
625+
ereport(ERROR,
626+
(errcode(ERRCODE_SYNTAX_ERROR),
627+
errmsg("duplicate column name \"%s\"", first_column.name),
628+
errhint("The sparse index option must reference distinct "
629+
"column.")));
630+
}
631+
*sparse_index_columns = ts_bmslist_add_set(*sparse_index_columns, attnums_bitmap);
632+
633+
firstlast_config.base = config;
634+
config_ptr = (SparseIndexConfigBase *) &firstlast_config;
635+
firstlast_config.col = first_column.name;
636+
break;
637+
617638
default:
618639
ereport(ERROR,
619640
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),

‎tsl/src/compression/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
set(SOURCES
22
${CMAKE_CURRENT_SOURCE_DIR}/api.c
33
${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_bloom1.c
4+
${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_firstlast.c
45
${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_minmax.c
56
${CMAKE_CURRENT_SOURCE_DIR}/compression.c
67
${CMAKE_CURRENT_SOURCE_DIR}/compression_dml.c

‎tsl/src/compression/batch_metadata_builder.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ enum BatchMetadataBuilderType
1414
{
1515
METADATA_BUILDER_MINMAX,
1616
METADATA_BUILDER_BLOOM1,
17+
METADATA_BUILDER_FIRSTLAST,
1718
};
1819

1920
typedef struct BatchMetadataBuilder
@@ -32,6 +33,10 @@ BatchMetadataBuilder *batch_metadata_builder_bloom1_create(int num_columns, cons
3233
const AttrNumber *attnums,
3334
int bloom_attr_offset);
3435

36+
BatchMetadataBuilder *batch_metadata_builder_firstlast_create(Oid type_oid, AttrNumber attnum,
37+
int first_attr_offset,
38+
int last_attr_offset);
39+
3540
/* Hasher interface common to bloom filters, used to compute the hash without updating the bloom
3641
* filter */
3742
typedef struct Bloom1Hasher

‎tsl/src/compression/batch_metadata_builder_firstlast.c‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
/*
2+
* This file and its contents are licensed under the Timescale License.
3+
* Please see the included NOTICE for copyright information and
4+
* LICENSE-TIMESCALE for a copy of the license.
5+
*/
6+
#include <postgres.h>
7+
#include <utils/datum.h>
8+
#include <utils/typcache.h>
9+
10+
#include "batch_metadata_builder_firstlast.h"
11+
12+
#include "compression.h"
13+
14+
static void firstlast_update_row(void *builder_, TupleTableSlot *slot);
15+
static void firstlast_insert_to_compressed_row(void *builder_, RowCompressor *compressor);
16+
static void firstlast_reset(void *builder_, RowCompressor *compressor);
17+
18+
BatchMetadataBuilder *
19+
batch_metadata_builder_firstlast_create(Oid type_oid, AttrNumber attnum, int first_attr_offset,
20+
int last_attr_offset)
21+
{
22+
BatchMetadataBuilderFirstLast *builder = palloc(sizeof(*builder));
23+
TypeCacheEntry *type = lookup_type_cache(type_oid, 0);
24+
25+
*builder = (BatchMetadataBuilderFirstLast){
26+
.functions =
27+
(BatchMetadataBuilder){
28+
.update_row = firstlast_update_row,
29+
.insert_to_compressed_row = firstlast_insert_to_compressed_row,
30+
.reset = firstlast_reset,
31+
.builder_type = METADATA_BUILDER_FIRSTLAST,
32+
},
33+
.type_oid = type_oid,
34+
.attnum = attnum,
35+
.empty = true,
36+
.type_by_val = type->typbyval,
37+
.type_len = type->typlen,
38+
.first_is_null = true,
39+
.last_is_null = true,
40+
.first_metadata_attr_offset = first_attr_offset,
41+
.last_metadata_attr_offset = last_attr_offset,
42+
};
43+
44+
return &builder->functions;
45+
}
46+
47+
static void
48+
firstlast_update_row(void *builder_, TupleTableSlot *slot)
49+
{
50+
BatchMetadataBuilderFirstLast *builder = (BatchMetadataBuilderFirstLast *) builder_;
51+
Assert(builder->functions.builder_type == METADATA_BUILDER_FIRSTLAST);
52+
53+
bool is_null;
54+
Datum val = slot_getattr(slot, builder->attnum, &is_null);
55+
56+
if (builder->empty)
57+
{
58+
if (is_null)
59+
{
60+
builder->first_is_null = true;
61+
builder->first = (Datum) 0;
62+
}
63+
else
64+
{
65+
builder->first_is_null = false;
66+
builder->first = datumCopy(val, builder->type_by_val, builder->type_len);
67+
}
68+
builder->empty = false;
69+
}
70+
71+
/* Always update last to the current row */
72+
if (!builder->last_is_null && !builder->type_by_val)
73+
pfree(DatumGetPointer(builder->last));
74+
75+
if (is_null)
76+
{
77+
builder->last_is_null = true;
78+
builder->last = (Datum) 0;
79+
}
80+
else
81+
{
82+
builder->last_is_null = false;
83+
builder->last = datumCopy(val, builder->type_by_val, builder->type_len);
84+
}
85+
}
86+
87+
static void
88+
firstlast_reset(void *builder_, RowCompressor *compressor)
89+
{
90+
BatchMetadataBuilderFirstLast *builder = (BatchMetadataBuilderFirstLast *) builder_;
91+
92+
if (!builder->empty)
93+
{
94+
if (!builder->first_is_null && !builder->type_by_val)
95+
pfree(DatumGetPointer(builder->first));
96+
if (!builder->last_is_null && !builder->type_by_val)
97+
pfree(DatumGetPointer(builder->last));
98+
builder->first = (Datum) 0;
99+
builder->last = (Datum) 0;
100+
}
101+
builder->empty = true;
102+
builder->first_is_null = true;
103+
builder->last_is_null = true;
104+
105+
compressor->compressed_is_null[builder->first_metadata_attr_offset] = true;
106+
compressor->compressed_is_null[builder->last_metadata_attr_offset] = true;
107+
compressor->compressed_values[builder->first_metadata_attr_offset] = (Datum) 0;
108+
compressor->compressed_values[builder->last_metadata_attr_offset] = (Datum) 0;
109+
}
110+
111+
static void
112+
firstlast_insert_to_compressed_row(void *builder_, RowCompressor *compressor)
113+
{
114+
BatchMetadataBuilderFirstLast *builder = (BatchMetadataBuilderFirstLast *) builder_;
115+
Assert(builder->first_metadata_attr_offset >= 0);
116+
Assert(builder->last_metadata_attr_offset >= 0);
117+
118+
if (builder->empty)
119+
{
120+
compressor->compressed_is_null[builder->first_metadata_attr_offset] = true;
121+
compressor->compressed_is_null[builder->last_metadata_attr_offset] = true;
122+
return;
123+
}
124+
125+
/* First value */
126+
if (builder->first_is_null)
127+
{
128+
compressor->compressed_is_null[builder->first_metadata_attr_offset] = true;
129+
}
130+
else
131+
{
132+
Datum first = builder->first;
133+
if (builder->type_len == -1)
134+
{
135+
Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(first));
136+
if (first != unpacked)
137+
pfree(DatumGetPointer(first));
138+
first = unpacked;
139+
builder->first = first;
140+
}
141+
compressor->compressed_is_null[builder->first_metadata_attr_offset] = false;
142+
compressor->compressed_values[builder->first_metadata_attr_offset] = first;
143+
}
144+
145+
/* Last value */
146+
if (builder->last_is_null)
147+
{
148+
compressor->compressed_is_null[builder->last_metadata_attr_offset] = true;
149+
}
150+
else
151+
{
152+
Datum last = builder->last;
153+
if (builder->type_len == -1)
154+
{
155+
Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(last));
156+
if (last != unpacked)
157+
pfree(DatumGetPointer(last));
158+
last = unpacked;
159+
builder->last = last;
160+
}
161+
compressor->compressed_is_null[builder->last_metadata_attr_offset] = false;
162+
compressor->compressed_values[builder->last_metadata_attr_offset] = last;
163+
}
164+
}

‎tsl/src/compression/batch_metadata_builder_firstlast.h‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* This file and its contents are licensed under the Timescale License.
3+
* Please see the included NOTICE for copyright information and
4+
* LICENSE-TIMESCALE for a copy of the license.
5+
*/
6+
#pragma once
7+
8+
#include <postgres.h>
9+
10+
#include "batch_metadata_builder.h"
11+
12+
typedef struct BatchMetadataBuilderFirstLast
13+
{
14+
BatchMetadataBuilder functions;
15+
16+
Oid type_oid;
17+
AttrNumber attnum;
18+
bool empty;
19+
20+
bool type_by_val;
21+
int16 type_len;
22+
23+
Datum first;
24+
bool first_is_null;
25+
26+
Datum last;
27+
bool last_is_null;
28+
29+
int16 first_metadata_attr_offset;
30+
int16 last_metadata_attr_offset;
31+
} BatchMetadataBuilderFirstLast;
32+
33+
typedef struct RowCompressor RowCompressor;

‎tsl/src/compression/compression.c‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -906,6 +906,31 @@ build_column_map(const CompressionSettings *settings, const TupleDesc in_desc,
906906
bloom_attr_offset));
907907
}
908908

909+
const AttrNumber first_attr_number =
910+
compressed_column_metadata_attno(settings,
911+
settings->fd.relid,
912+
attr->attnum,
913+
settings->fd.compress_relid,
914+
"first");
915+
const AttrNumber last_attr_number =
916+
compressed_column_metadata_attno(settings,
917+
settings->fd.relid,
918+
attr->attnum,
919+
settings->fd.compress_relid,
920+
"last");
921+
if (AttributeNumberIsValid(first_attr_number) &&
922+
AttributeNumberIsValid(last_attr_number))
923+
{
924+
const int16 first_attr_offset = AttrNumberGetAttrOffset(first_attr_number);
925+
const int16 last_attr_offset = AttrNumberGetAttrOffset(last_attr_number);
926+
metadata_builders =
927+
lappend(metadata_builders,
928+
batch_metadata_builder_firstlast_create(attr->atttypid,
929+
attr->attnum,
930+
first_attr_offset,
931+
last_attr_offset));
932+
}
933+
909934
*column = (PerColumn){
910935
.compressor = compressor_for_type(attr->atttypid),
911936
.segmentby_column_index = -1,

0 commit comments

Comments
 (0)