Skip to content

Commit 813e390

Browse files
iceberg: fix decimal min/max stats extraction for parquet files (#4368)
1 parent 757dbcd commit 813e390

3 files changed

Lines changed: 78 additions & 1 deletion

File tree

CHANGELOG.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,11 @@ Changelog
33

44
All notable changes to this project will be documented in this file.
55

6-
## Unreleased
6+
## 4.89.3 - 2026-04-30
7+
8+
### Fixed
9+
10+
- iceberg: fix decimal min/max stats extraction for parquet files ([@josephwoodward](https://github.com/josephwoodward), [#4368](https://github.com/redpanda-data/connect/pull/4368))
711

812
### Added
913

internal/impl/iceberg/icebergx/stats.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,16 @@ func goValueToLiteral(val any, iceType iceberg.Type) (iceberg.Literal, error) {
293293
return nil, fmt.Errorf("fixed type requires %d bytes, got %d", t.Len(), len(b))
294294
}
295295
return iceberg.NewLiteral(b[:t.Len()]), nil
296+
case iceberg.DecimalType:
297+
// Parquet stores decimals as big-endian two's complement (FIXED_LEN_BYTE_ARRAY).
298+
// iceberg.DecimalLiteral.UnmarshalBinary expects the same encoding.
299+
b := val.([]byte)
300+
var lit iceberg.DecimalLiteral
301+
if err := lit.UnmarshalBinary(b); err != nil {
302+
return nil, fmt.Errorf("decoding decimal value: %w", err)
303+
}
304+
lit.Scale = t.Scale()
305+
return lit, nil
296306
default:
297307
return nil, fmt.Errorf("unsupported iceberg type: %v", iceType)
298308
}
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright 2026 Redpanda Data, Inc.
3+
*
4+
* Licensed as a Redpanda Enterprise file under the Redpanda Community
5+
* License (the "License"); you may not use this file except in compliance with
6+
* the License. You may obtain a copy of the License at
7+
*
8+
* https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md
9+
*/
10+
11+
package icebergx
12+
13+
import (
14+
"testing"
15+
16+
"github.com/apache/iceberg-go"
17+
"github.com/stretchr/testify/assert"
18+
"github.com/stretchr/testify/require"
19+
)
20+
21+
func TestGoValueToLiteralDecimal(t *testing.T) {
22+
tests := []struct {
23+
name string
24+
bytes []byte
25+
iceType iceberg.Type
26+
wantStr string
27+
wantScale int
28+
}{
29+
{
30+
name: "positive decimal(20,0)",
31+
bytes: []byte{0x30, 0x39}, // 12345 big-endian two's complement
32+
iceType: iceberg.DecimalTypeOf(20, 0),
33+
wantStr: "12345",
34+
wantScale: 0,
35+
},
36+
{
37+
name: "negative decimal(10,2)",
38+
bytes: []byte{0xFF}, // -1 unscaled, scale 2 => -0.01
39+
iceType: iceberg.DecimalTypeOf(10, 2),
40+
wantStr: "-0.01",
41+
wantScale: 2,
42+
},
43+
{
44+
name: "zero decimal(5,0)",
45+
bytes: []byte{0x00},
46+
iceType: iceberg.DecimalTypeOf(5, 0),
47+
wantStr: "0",
48+
wantScale: 0,
49+
},
50+
}
51+
52+
for _, tt := range tests {
53+
t.Run(tt.name, func(t *testing.T) {
54+
lit, err := goValueToLiteral(tt.bytes, tt.iceType)
55+
require.NoError(t, err)
56+
57+
decLit, ok := lit.(iceberg.DecimalLiteral)
58+
require.True(t, ok, "expected iceberg.DecimalLiteral, got %T", lit)
59+
assert.Equal(t, tt.wantScale, decLit.Scale)
60+
assert.Equal(t, tt.wantStr, lit.String())
61+
})
62+
}
63+
}

0 commit comments

Comments
 (0)