Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit a8f029b

Browse files
authored
Fix creating Nullable(Tuple) type (#1282)
This is an invalid ClickHouse type which can end up with: ``` error executing query: code: 43, message: Nested type Tuple(id Nullable(Int64), ns Nullable(Int64)) cannot be inside Nullable type ``` Tuples, similarly to Arrays, cannot be wrapped in a `Nullable` type. This PR fixes this behavior and adds relevant test.
1 parent 2bb2542 commit a8f029b

File tree

3 files changed

+58
-2
lines changed

3 files changed

+58
-2
lines changed

quesma/clickhouse/schema.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ func (t MultiValueType) String() string {
130130
// WORKAROUND: if col.Name == "discount_amount" || col.Name == "unit_discount_amount" -> tupleParams = append(tupleParams, fmt.Sprintf("%s %s", col.Name, "Float64"))
131131
// But it's not a good solution, need to find a better one
132132
colType := col.Type.String()
133-
if !strings.Contains(colType, "Array") && !strings.Contains(colType, "DateTime") {
133+
if (!strings.Contains(colType, "Array") && !strings.Contains(colType, "Tuple")) && !strings.Contains(colType, "DateTime") {
134134
colType = "Nullable(" + colType + ")"
135135
}
136136
tupleParams = append(tupleParams, fmt.Sprintf("%s %s", col.Name, colType))

quesma/ingest/parser.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ func JsonToColumns(m SchemaMap, chConfig *clickhouse.ChTableConfig) []CreateTabl
9898
}
9999

100100
fTypeString := fType.String()
101-
if !strings.Contains(fTypeString, "Array") && !strings.Contains(fTypeString, "DateTime") {
101+
if (!strings.Contains(fTypeString, "Array") && !strings.Contains(fTypeString, "Tuple")) && !strings.Contains(fTypeString, "DateTime") {
102102
fTypeString = "Nullable(" + fTypeString + ")"
103103
}
104104

quesma/ingest/parser_test.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright Quesma, licensed under the Elastic License 2.0.
2+
// SPDX-License-Identifier: Elastic-2.0
3+
4+
package ingest
5+
6+
import (
7+
"github.com/QuesmaOrg/quesma/quesma/clickhouse"
8+
"github.com/QuesmaOrg/quesma/quesma/quesma/types"
9+
"github.com/stretchr/testify/assert"
10+
"testing"
11+
)
12+
13+
func TestJsonToColumns(t *testing.T) {
14+
tests := []struct {
15+
name string
16+
payload string
17+
expectedColumnName string
18+
expectedTypeString []string // Tuple elements order is non-deterministic
19+
}{
20+
{
21+
name: "NestedTuples",
22+
payload: `{"nested_tuples":[[{"test":{"id":"asdf","ns":"rrrr"}}]]}`,
23+
24+
expectedColumnName: "nested_tuples",
25+
expectedTypeString: []string{"Array(Array(Tuple(test Tuple(id Nullable(String), ns Nullable(String)))))",
26+
"Array(Array(Tuple(test Tuple(ns Nullable(String), id Nullable(String)))))"},
27+
},
28+
{
29+
name: "NotSoDeeplyNested",
30+
payload: `{"not_so_deeply_nested":[{"test":{"id":0.1337,"ns":1233}}]}`,
31+
32+
expectedColumnName: "not_so_deeply_nested",
33+
expectedTypeString: []string{"Array(Tuple(test Tuple(id Nullable(Float64), ns Nullable(Int64))))",
34+
"Array(Tuple(test Tuple(ns Nullable(Int64), id Nullable(Float64))))"},
35+
},
36+
{
37+
name: "Timestamp",
38+
payload: `{"@timestamp":"2024-01-27T16:11:19.94Z"}`,
39+
expectedColumnName: "@timestamp",
40+
expectedTypeString: []string{"DateTime64 DEFAULT now64()"},
41+
},
42+
}
43+
44+
for _, tt := range tests {
45+
t.Run(tt.name, func(t *testing.T) {
46+
json, _ := types.ParseJSON(tt.payload)
47+
48+
result := JsonToColumns(json, &clickhouse.ChTableConfig{
49+
TimestampDefaultsNow: true,
50+
})
51+
52+
assert.Equal(t, tt.expectedColumnName, result[0].ClickHouseColumnName)
53+
assert.Contains(t, tt.expectedTypeString, result[0].ClickHouseType)
54+
})
55+
}
56+
}

0 commit comments

Comments
 (0)