Skip to content

Commit e4f7774

Browse files
antekresicantekresic
committed
Add firstlast sparse index type for compression
Add a new sparse index type that stores the actual first and last values of each compressed batch in sort order, including NULLs. Unlike minmax, which skips NULLs and tracks statistical extremes, firstlast tracks positional boundary values. This enables tracking batch ordering and NULL presence at batch boundaries.
1 parent f8fc88a commit e4f7774

13 files changed

Lines changed: 872 additions & 13 deletions

‎.unreleased/pr_9580‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Implements: #9580 Add first/last sparse indexes to compression

‎src/ts_catalog/compression_settings.c‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#include <common/md5.h>
2121
#include <utils/palloc.h>
2222

23-
TSDLLEXPORT const char *ts_sparse_index_type_names[] = { "bloom", "minmax" };
23+
TSDLLEXPORT const char *ts_sparse_index_type_names[] = { "bloom", "minmax", "firstlast" };
2424
TSDLLEXPORT const char *ts_sparse_index_source_names[] = { "config", "default", "orderby" };
2525
TSDLLEXPORT const char *ts_sparse_index_common_keys[] = { "type", "column", "source", NULL };
2626
static ScanTupleResult compression_settings_tuple_update(TupleInfo *ti, void *data);
@@ -628,6 +628,14 @@ ts_convert_sparse_index_config_to_jsonb(JsonbParseState *parse_state, SparseInde
628628
ts_sparse_index_common_keys[SparseIndexKeyCol],
629629
minmax_config->col); /* column */
630630
break;
631+
case _SparseIndexTypeEnumFirstLast:
632+
{
633+
FirstLastIndexColumnConfig *firstlast_config = (FirstLastIndexColumnConfig *) config;
634+
ts_jsonb_add_str(parse_state,
635+
ts_sparse_index_common_keys[SparseIndexKeyCol],
636+
firstlast_config->col); /* column */
637+
break;
638+
}
631639
case _SparseIndexTypeEnumBloom:
632640
bloom_config = (BloomFilterConfig *) config;
633641

@@ -1403,6 +1411,7 @@ ts_get_per_column_compression_settings(const SparseIndexSettings *settings)
14031411
per_column_setting = palloc0(sizeof(PerColumnCompressionSettings));
14041412
per_column_setting->column_name = column_name;
14051413
per_column_setting->minmax_obj_id = -1;
1414+
per_column_setting->firstlast_obj_id = -1;
14061415
per_column_setting->single_bloom_obj_id = -1;
14071416
per_column_setting->composite_bloom_index_obj_ids = NULL;
14081417
result_settings = lappend(result_settings, per_column_setting);
@@ -1413,6 +1422,12 @@ ts_get_per_column_compression_settings(const SparseIndexSettings *settings)
14131422
Assert(num_columns == 1);
14141423
per_column_setting->minmax_obj_id = obj_id;
14151424
}
1425+
else if (strcmp(index_type,
1426+
ts_sparse_index_type_names[_SparseIndexTypeEnumFirstLast]) == 0)
1427+
{
1428+
Assert(num_columns == 1);
1429+
per_column_setting->firstlast_obj_id = obj_id;
1430+
}
14161431
else if (strcmp(index_type,
14171432
ts_sparse_index_type_names[_SparseIndexTypeEnumBloom]) == 0)
14181433
{

‎src/ts_catalog/compression_settings.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ typedef enum SparseIndexTypeEnum
2020
{
2121
_SparseIndexTypeEnumBloom = 0,
2222
_SparseIndexTypeEnumMinmax,
23+
_SparseIndexTypeEnumFirstLast,
2324
_SparseIndexTypeEnumMax
2425
} SparseIndexTypeEnum;
2526

@@ -55,6 +56,12 @@ typedef struct MinmaxIndexColumnConfig
5556
const char *col;
5657
} MinmaxIndexColumnConfig;
5758

59+
typedef struct FirstLastIndexColumnConfig
60+
{
61+
SparseIndexConfigBase base;
62+
const char *col;
63+
} FirstLastIndexColumnConfig;
64+
5865
typedef struct SparseIndexColumn
5966
{
6067
/* composite bloom indexes will have multiple SparseIndexColumn entries and
@@ -116,6 +123,9 @@ typedef struct PerColumnCompressionSettings
116123
/* the index of the minmax index object that the column participates in, -1 if not present */
117124
int minmax_obj_id;
118125

126+
/* the index of the firstlast index object that the column participates in, -1 if not present */
127+
int firstlast_obj_id;
128+
119129
/* the index of the single bloom index object that the column participates in, -1 if not present
120130
*/
121131
int single_bloom_obj_id;

‎src/with_clause/alter_table_with_clause.c‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ static const WithClauseDefinition sparse_index_with_clause_def[] = {
6868
.arg_names = {"compress_minmax", "minmax", "compress_min_max", "min_max", NULL},
6969
.type_id = TEXTOID,
7070
},
71+
[_SparseIndexTypeEnumFirstLast] = {
72+
.arg_names = {"compress_firstlast", "firstlast", "first_last", NULL},
73+
.type_id = TEXTOID,
74+
},
7175
};
7276

7377
WithClauseResult *
@@ -535,6 +539,7 @@ parse_sparse_index_config(JsonbParseState *parse_state, FuncCall *sparse_index_d
535539
{
536540
TypeCacheEntry *type_cache;
537541
MinmaxIndexColumnConfig minmax_config;
542+
FirstLastIndexColumnConfig firstlast_config;
538543
BloomFilterConfig bloom_config;
539544
SparseIndexConfigBase config;
540545
SparseIndexConfigBase *config_ptr = &config;
@@ -573,7 +578,7 @@ parse_sparse_index_config(JsonbParseState *parse_state, FuncCall *sparse_index_d
573578
{
574579
ereport(ERROR,
575580
(errcode(ERRCODE_SYNTAX_ERROR),
576-
errmsg("minmax index can only have one column")));
581+
errmsg("only bloom indexes can have multiple columns")));
577582
}
578583
}
579584

@@ -682,6 +687,22 @@ parse_sparse_index_config(JsonbParseState *parse_state, FuncCall *sparse_index_d
682687
minmax_config.col = first_column.name;
683688
break;
684689

690+
case _SparseIndexTypeEnumFirstLast:
691+
if (ts_bmslist_contains_set(*sparse_index_columns, attnums_bitmap))
692+
{
693+
ereport(ERROR,
694+
(errcode(ERRCODE_SYNTAX_ERROR),
695+
errmsg("duplicate column name \"%s\"", first_column.name),
696+
errhint("The sparse index option must reference distinct "
697+
"column.")));
698+
}
699+
*sparse_index_columns = ts_bmslist_add_set(*sparse_index_columns, attnums_bitmap);
700+
701+
firstlast_config.base = config;
702+
config_ptr = (SparseIndexConfigBase *) &firstlast_config;
703+
firstlast_config.col = first_column.name;
704+
break;
705+
685706
default:
686707
ereport(ERROR,
687708
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),

‎tsl/src/compression/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
set(SOURCES
22
${CMAKE_CURRENT_SOURCE_DIR}/api.c
33
${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_bloom1.c
4+
${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_firstlast.c
45
${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_minmax.c
56
${CMAKE_CURRENT_SOURCE_DIR}/compression.c
67
${CMAKE_CURRENT_SOURCE_DIR}/compression_dml.c

‎tsl/src/compression/batch_metadata_builder.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ enum BatchMetadataBuilderType
1414
{
1515
METADATA_BUILDER_MINMAX,
1616
METADATA_BUILDER_BLOOM1,
17+
METADATA_BUILDER_FIRSTLAST,
1718
};
1819

1920
typedef struct BatchMetadataBuilder
@@ -32,6 +33,10 @@ BatchMetadataBuilder *batch_metadata_builder_bloom1_create(int num_columns, cons
3233
const AttrNumber *attnums,
3334
int bloom_attr_offset);
3435

36+
BatchMetadataBuilder *batch_metadata_builder_firstlast_create(Oid type_oid, AttrNumber attnum,
37+
int first_attr_offset,
38+
int last_attr_offset);
39+
3540
/* Hasher interface common to bloom filters, used to compute the hash without updating the bloom
3641
* filter */
3742
typedef struct Bloom1Hasher

‎tsl/src/compression/batch_metadata_builder_firstlast.c‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
/*
2+
* This file and its contents are licensed under the Timescale License.
3+
* Please see the included NOTICE for copyright information and
4+
* LICENSE-TIMESCALE for a copy of the license.
5+
*/
6+
#include <postgres.h>
7+
#include <utils/datum.h>
8+
#include <utils/typcache.h>
9+
10+
#include "batch_metadata_builder_firstlast.h"
11+
12+
#include "compression.h"
13+
14+
static void firstlast_update_row(void *builder_, TupleTableSlot *slot);
15+
static void firstlast_insert_to_compressed_row(void *builder_, RowCompressor *compressor);
16+
static void firstlast_reset(void *builder_, RowCompressor *compressor);
17+
18+
BatchMetadataBuilder *
19+
batch_metadata_builder_firstlast_create(Oid type_oid, AttrNumber attnum, int first_attr_offset,
20+
int last_attr_offset)
21+
{
22+
BatchMetadataBuilderFirstLast *builder = palloc(sizeof(*builder));
23+
TypeCacheEntry *type = lookup_type_cache(type_oid, 0);
24+
25+
*builder = (BatchMetadataBuilderFirstLast){
26+
.functions =
27+
(BatchMetadataBuilder){
28+
.update_row = firstlast_update_row,
29+
.insert_to_compressed_row = firstlast_insert_to_compressed_row,
30+
.reset = firstlast_reset,
31+
.builder_type = METADATA_BUILDER_FIRSTLAST,
32+
},
33+
.type_oid = type_oid,
34+
.attnum = attnum,
35+
.empty = true,
36+
.type_by_val = type->typbyval,
37+
.type_len = type->typlen,
38+
.first_is_null = true,
39+
.last_is_null = true,
40+
.first_metadata_attr_offset = first_attr_offset,
41+
.last_metadata_attr_offset = last_attr_offset,
42+
};
43+
44+
return &builder->functions;
45+
}
46+
47+
static void
48+
firstlast_update_row(void *builder_, TupleTableSlot *slot)
49+
{
50+
BatchMetadataBuilderFirstLast *builder = (BatchMetadataBuilderFirstLast *) builder_;
51+
Assert(builder->functions.builder_type == METADATA_BUILDER_FIRSTLAST);
52+
53+
bool is_null;
54+
Datum val = slot_getattr(slot, builder->attnum, &is_null);
55+
56+
if (builder->empty)
57+
{
58+
if (is_null)
59+
{
60+
builder->first_is_null = true;
61+
builder->first = (Datum) 0;
62+
}
63+
else
64+
{
65+
builder->first_is_null = false;
66+
builder->first = datumCopy(val, builder->type_by_val, builder->type_len);
67+
}
68+
builder->empty = false;
69+
}
70+
71+
/* Always update last to the current row */
72+
if (!builder->last_is_null && !builder->type_by_val)
73+
{
74+
pfree(DatumGetPointer(builder->last));
75+
}
76+
77+
if (is_null)
78+
{
79+
builder->last_is_null = true;
80+
builder->last = (Datum) 0;
81+
}
82+
else
83+
{
84+
builder->last_is_null = false;
85+
builder->last = datumCopy(val, builder->type_by_val, builder->type_len);
86+
}
87+
}
88+
89+
static void
90+
firstlast_reset(void *builder_, RowCompressor *compressor)
91+
{
92+
BatchMetadataBuilderFirstLast *builder = (BatchMetadataBuilderFirstLast *) builder_;
93+
94+
if (!builder->empty)
95+
{
96+
if (!builder->first_is_null && !builder->type_by_val)
97+
{
98+
pfree(DatumGetPointer(builder->first));
99+
}
100+
if (!builder->last_is_null && !builder->type_by_val)
101+
{
102+
pfree(DatumGetPointer(builder->last));
103+
}
104+
builder->first = (Datum) 0;
105+
builder->last = (Datum) 0;
106+
}
107+
builder->empty = true;
108+
builder->first_is_null = true;
109+
builder->last_is_null = true;
110+
111+
compressor->compressed_is_null[builder->first_metadata_attr_offset] = true;
112+
compressor->compressed_is_null[builder->last_metadata_attr_offset] = true;
113+
compressor->compressed_values[builder->first_metadata_attr_offset] = (Datum) 0;
114+
compressor->compressed_values[builder->last_metadata_attr_offset] = (Datum) 0;
115+
}
116+
117+
static void
118+
firstlast_insert_to_compressed_row(void *builder_, RowCompressor *compressor)
119+
{
120+
BatchMetadataBuilderFirstLast *builder = (BatchMetadataBuilderFirstLast *) builder_;
121+
Assert(builder->first_metadata_attr_offset >= 0);
122+
Assert(builder->last_metadata_attr_offset >= 0);
123+
124+
if (builder->empty)
125+
{
126+
compressor->compressed_is_null[builder->first_metadata_attr_offset] = true;
127+
compressor->compressed_is_null[builder->last_metadata_attr_offset] = true;
128+
return;
129+
}
130+
131+
/* First value */
132+
if (builder->first_is_null)
133+
{
134+
compressor->compressed_is_null[builder->first_metadata_attr_offset] = true;
135+
}
136+
else
137+
{
138+
Datum first = builder->first;
139+
if (builder->type_len == -1)
140+
{
141+
Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(first));
142+
if (first != unpacked)
143+
{
144+
pfree(DatumGetPointer(first));
145+
}
146+
first = unpacked;
147+
builder->first = first;
148+
}
149+
compressor->compressed_is_null[builder->first_metadata_attr_offset] = false;
150+
compressor->compressed_values[builder->first_metadata_attr_offset] = first;
151+
}
152+
153+
/* Last value */
154+
if (builder->last_is_null)
155+
{
156+
compressor->compressed_is_null[builder->last_metadata_attr_offset] = true;
157+
}
158+
else
159+
{
160+
Datum last = builder->last;
161+
if (builder->type_len == -1)
162+
{
163+
Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(last));
164+
if (last != unpacked)
165+
{
166+
pfree(DatumGetPointer(last));
167+
}
168+
last = unpacked;
169+
builder->last = last;
170+
}
171+
compressor->compressed_is_null[builder->last_metadata_attr_offset] = false;
172+
compressor->compressed_values[builder->last_metadata_attr_offset] = last;
173+
}
174+
}

‎tsl/src/compression/batch_metadata_builder_firstlast.h‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* This file and its contents are licensed under the Timescale License.
3+
* Please see the included NOTICE for copyright information and
4+
* LICENSE-TIMESCALE for a copy of the license.
5+
*/
6+
#pragma once
7+
8+
#include <postgres.h>
9+
10+
#include "batch_metadata_builder.h"
11+
12+
typedef struct BatchMetadataBuilderFirstLast
13+
{
14+
BatchMetadataBuilder functions;
15+
16+
Oid type_oid;
17+
AttrNumber attnum;
18+
bool empty;
19+
20+
bool type_by_val;
21+
int16 type_len;
22+
23+
Datum first;
24+
bool first_is_null;
25+
26+
Datum last;
27+
bool last_is_null;
28+
29+
int16 first_metadata_attr_offset;
30+
int16 last_metadata_attr_offset;
31+
} BatchMetadataBuilderFirstLast;
32+
33+
typedef struct RowCompressor RowCompressor;

‎tsl/src/compression/compression.c‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -960,6 +960,31 @@ build_column_map(const CompressionSettings *settings, const TupleDesc in_desc,
960960
bloom_attr_offset));
961961
}
962962

963+
const AttrNumber first_attr_number =
964+
compressed_column_metadata_attno(settings,
965+
settings->fd.relid,
966+
attr->attnum,
967+
settings->fd.compress_relid,
968+
"first");
969+
const AttrNumber last_attr_number =
970+
compressed_column_metadata_attno(settings,
971+
settings->fd.relid,
972+
attr->attnum,
973+
settings->fd.compress_relid,
974+
"last");
975+
if (AttributeNumberIsValid(first_attr_number) &&
976+
AttributeNumberIsValid(last_attr_number))
977+
{
978+
const int16 first_attr_offset = AttrNumberGetAttrOffset(first_attr_number);
979+
const int16 last_attr_offset = AttrNumberGetAttrOffset(last_attr_number);
980+
metadata_builders =
981+
lappend(metadata_builders,
982+
batch_metadata_builder_firstlast_create(attr->atttypid,
983+
attr->attnum,
984+
first_attr_offset,
985+
last_attr_offset));
986+
}
987+
963988
*column = (PerColumn){
964989
.compressor = compressor_for_type(attr->atttypid),
965990
.segmentby_column_index = -1,

0 commit comments

Comments
 (0)