Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit 2506adb

Browse files
jakozaurjakozaur
authored
Combine ALTER TABLE into one (#1463)
`ALTER TABLE` can be expensive in ClickHouse, it's easier and better to combine them. The MVP is to combine adding a new column with adding a comment to it.
1 parent 0326054 commit 2506adb

File tree

4 files changed

+18
-34
lines changed

4 files changed

+18
-34
lines changed

‎platform/ingest/alter_table_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ func TestAlterTable(t *testing.T) {
3636
"{\"Test1\":1,\"Test2\":2}",
3737
}
3838
alters := []string{
39-
"ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test1\" Nullable(Int64)",
40-
"ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test2\" Nullable(Int64)",
39+
"ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test1\" Nullable(Int64), COMMENT COLUMN \"Test1\" 'quesmaMetadataV1:fieldName=Test1'",
40+
"ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test2\" Nullable(Int64), COMMENT COLUMN \"Test2\" 'quesmaMetadataV1:fieldName=Test2'",
4141
}
4242
columns := []string{"Test1", "Test2"}
4343
table := &clickhouse.Table{

‎platform/ingest/common_table_test.go‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,7 @@ func TestIngestToCommonTable(t *testing.T) {
3535
{"foo": "bar"},
3636
},
3737
expectedStatements: []string{
38-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`,
39-
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
38+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
4039
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}`,
4140
},
4241
virtualTableColumns: []string{"@timestamp", "foo"},
@@ -48,8 +47,7 @@ func TestIngestToCommonTable(t *testing.T) {
4847
{"foo": "baz"},
4948
},
5049
expectedStatements: []string{
51-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`,
52-
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
50+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
5351
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}`,
5452
},
5553
virtualTableColumns: []string{"@timestamp", "foo"},
@@ -62,10 +60,8 @@ func TestIngestToCommonTable(t *testing.T) {
6260
{"foo": "1", "baz": "qux"},
6361
},
6462
expectedStatements: []string{
65-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`,
66-
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
67-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String)`,
68-
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`,
63+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
64+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String), COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`,
6965

7066
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}, {"__quesma_index_name":"test_index","baz":"qux","foo":"1"} `,
7167
},
@@ -82,10 +78,8 @@ func TestIngestToCommonTable(t *testing.T) {
8278
{"foo": "1", "baz": "qux"},
8379
},
8480
expectedStatements: []string{
85-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`,
86-
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
87-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String)`,
88-
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`,
81+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
82+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String), COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`,
8983

9084
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}, {"__quesma_index_name":"test_index","baz":"qux","foo":"1"} `,
9185
},
@@ -113,8 +107,7 @@ func TestIngestToCommonTable(t *testing.T) {
113107
{"a": "bar", "b": "baz"},
114108
},
115109
expectedStatements: []string{
116-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "b" Nullable(String)`,
117-
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "b" 'quesmaMetadataV1:fieldName=b'`,
110+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "b" Nullable(String), COMMENT COLUMN "b" 'quesmaMetadataV1:fieldName=b'`,
118111

119112
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","a":"bar","b":"baz"}`,
120113
},
@@ -127,8 +120,7 @@ func TestIngestToCommonTable(t *testing.T) {
127120
{"a.b": "c"},
128121
},
129122
expectedStatements: []string{
130-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "a_b" Nullable(String)`,
131-
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "a_b" 'quesmaMetadataV1:fieldName=a.b'`,
123+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "a_b" Nullable(String), COMMENT COLUMN "a_b" 'quesmaMetadataV1:fieldName=a.b'`,
132124

133125
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","a_b":"c"}`,
134126
},

‎platform/ingest/insert_test.go‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -102,27 +102,21 @@ var configs = []*clickhouse.ChTableConfig{
102102
var expectedInserts = [][]string{
103103
[]string{EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","service_name":"frontend","severity":"debug","source":"rhel"}`)},
104104
[]string{
105-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "service_name" Nullable(String)`),
106-
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "service_name" 'quesmaMetadataV1:fieldName=service.name`),
105+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "service_name" Nullable(String), COMMENT COLUMN "service_name" 'quesmaMetadataV1:fieldName=service.name'`),
107106

108-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String)`),
109-
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`),
107+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String), COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`),
110108

111-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "source" Nullable(String)`),
112-
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "source" 'quesmaMetadataV1:fieldName=source'`),
109+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "source" Nullable(String), COMMENT COLUMN "source" 'quesmaMetadataV1:fieldName=source'`),
113110

114111
EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","service_name":"frontend","severity":"debug","source":"rhel"}`),
115112
},
116113
[]string{
117114
EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","random1":["debug"],"random2":"random-string","severity":"frontend"}`),
118115
},
119116
[]string{
120-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random1" Array(String)`),
121-
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "random1" 'quesmaMetadataV1:fieldName=random1'`),
122-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random2" Nullable(String)`),
123-
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "random2" 'quesmaMetadataV1:fieldName=random2'`),
124-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String)`),
125-
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`),
117+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random1" Array(String), COMMENT COLUMN "random1" 'quesmaMetadataV1:fieldName=random1'`),
118+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random2" Nullable(String), COMMENT COLUMN "random2" 'quesmaMetadataV1:fieldName=random2'`),
119+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String), COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`),
126120
EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","random1":["debug"],"random2":"random-string","severity":"frontend"}`),
127121
},
128122
}

‎platform/ingest/processor.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -382,13 +382,11 @@ func (ip *IngestProcessor) generateNewColumns(
382382
maybeOnClusterClause = " ON CLUSTER " + strconv.Quote(table.ClusterName)
383383
}
384384

385-
alterTable := fmt.Sprintf("ALTER TABLE \"%s\"%s ADD COLUMN IF NOT EXISTS \"%s\" %s", table.Name, maybeOnClusterClause, attrKeys[i], columnType)
385+
alterTable := fmt.Sprintf(`ALTER TABLE "%s"%s ADD COLUMN IF NOT EXISTS "%s" %s, COMMENT COLUMN "%s" '%s'`,
386+
table.Name, maybeOnClusterClause, attrKeys[i], columnType, attrKeys[i], comment)
386387
newColumns[attrKeys[i]] = &chLib.Column{Name: attrKeys[i], Type: chLib.NewBaseType(attrTypes[i]), Modifiers: modifiers, Comment: comment}
387388
alterCmd = append(alterCmd, alterTable)
388389

389-
alterColumn := fmt.Sprintf("ALTER TABLE \"%s\"%s COMMENT COLUMN \"%s\" '%s'", table.Name, maybeOnClusterClause, attrKeys[i], comment)
390-
alterCmd = append(alterCmd, alterColumn)
391-
392390
deleteIndexes = append(deleteIndexes, i)
393391
}
394392

0 commit comments

Comments
 (0)