Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit 5b118a9

Browse files
nablaonenablaone
authored
Revert "Combine ALTER TABLE into one (#1463)" (#1476)
This reverts commit 2506adb. Combined alter table statements may not always work. ``` Error executing ingest statement: code: 48, message: For Replicated databases it's not allowed to execute ALTERs of different types (replicated and non replicated) in single query, query: ALTER TABLE "phone_home_logs" ADD COLUMN IF NOT EXISTS "request_id" Nullable(String), COMMENT COLUMN... ``` <!-- A note on testing your PR --> <!-- Basic unit test run is executed against each commit in the PR. If you want to run a full integration test suite, you can trigger it by commenting with '/run-integration-tests' or '/run-it' -->
1 parent c6ee5bd commit 5b118a9

File tree

4 files changed

+34
-18
lines changed

4 files changed

+34
-18
lines changed

‎platform/ingest/alter_table_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ func TestAlterTable(t *testing.T) {
3636
"{\"Test1\":1,\"Test2\":2}",
3737
}
3838
alters := []string{
39-
"ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test1\" Nullable(Int64), COMMENT COLUMN \"Test1\" 'quesmaMetadataV1:fieldName=Test1'",
40-
"ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test2\" Nullable(Int64), COMMENT COLUMN \"Test2\" 'quesmaMetadataV1:fieldName=Test2'",
39+
"ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test1\" Nullable(Int64)",
40+
"ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test2\" Nullable(Int64)",
4141
}
4242
columns := []string{"Test1", "Test2"}
4343
table := &clickhouse.Table{

‎platform/ingest/common_table_test.go‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ func TestIngestToCommonTable(t *testing.T) {
3535
{"foo": "bar"},
3636
},
3737
expectedStatements: []string{
38-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
38+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`,
39+
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
3940
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}`,
4041
},
4142
virtualTableColumns: []string{"@timestamp", "foo"},
@@ -47,7 +48,8 @@ func TestIngestToCommonTable(t *testing.T) {
4748
{"foo": "baz"},
4849
},
4950
expectedStatements: []string{
50-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
51+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`,
52+
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
5153
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}`,
5254
},
5355
virtualTableColumns: []string{"@timestamp", "foo"},
@@ -60,8 +62,10 @@ func TestIngestToCommonTable(t *testing.T) {
6062
{"foo": "1", "baz": "qux"},
6163
},
6264
expectedStatements: []string{
63-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
64-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String), COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`,
65+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`,
66+
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
67+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String)`,
68+
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`,
6569

6670
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}, {"__quesma_index_name":"test_index","baz":"qux","foo":"1"} `,
6771
},
@@ -78,8 +82,10 @@ func TestIngestToCommonTable(t *testing.T) {
7882
{"foo": "1", "baz": "qux"},
7983
},
8084
expectedStatements: []string{
81-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
82-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String), COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`,
85+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`,
86+
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`,
87+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String)`,
88+
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`,
8389

8490
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}, {"__quesma_index_name":"test_index","baz":"qux","foo":"1"} `,
8591
},
@@ -107,7 +113,8 @@ func TestIngestToCommonTable(t *testing.T) {
107113
{"a": "bar", "b": "baz"},
108114
},
109115
expectedStatements: []string{
110-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "b" Nullable(String), COMMENT COLUMN "b" 'quesmaMetadataV1:fieldName=b'`,
116+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "b" Nullable(String)`,
117+
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "b" 'quesmaMetadataV1:fieldName=b'`,
111118

112119
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","a":"bar","b":"baz"}`,
113120
},
@@ -120,7 +127,8 @@ func TestIngestToCommonTable(t *testing.T) {
120127
{"a.b": "c"},
121128
},
122129
expectedStatements: []string{
123-
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "a_b" Nullable(String), COMMENT COLUMN "a_b" 'quesmaMetadataV1:fieldName=a.b'`,
130+
`ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "a_b" Nullable(String)`,
131+
`ALTER TABLE "quesma_common_table" COMMENT COLUMN "a_b" 'quesmaMetadataV1:fieldName=a.b'`,
124132

125133
`INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","a_b":"c"}`,
126134
},

‎platform/ingest/insert_test.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,21 +102,27 @@ var configs = []*clickhouse.ChTableConfig{
102102
var expectedInserts = [][]string{
103103
[]string{EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","service_name":"frontend","severity":"debug","source":"rhel"}`)},
104104
[]string{
105-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "service_name" Nullable(String), COMMENT COLUMN "service_name" 'quesmaMetadataV1:fieldName=service.name'`),
105+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "service_name" Nullable(String)`),
106+
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "service_name" 'quesmaMetadataV1:fieldName=service.name`),
106107

107-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String), COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`),
108+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String)`),
109+
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`),
108110

109-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "source" Nullable(String), COMMENT COLUMN "source" 'quesmaMetadataV1:fieldName=source'`),
111+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "source" Nullable(String)`),
112+
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "source" 'quesmaMetadataV1:fieldName=source'`),
110113

111114
EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","service_name":"frontend","severity":"debug","source":"rhel"}`),
112115
},
113116
[]string{
114117
EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","random1":["debug"],"random2":"random-string","severity":"frontend"}`),
115118
},
116119
[]string{
117-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random1" Array(String), COMMENT COLUMN "random1" 'quesmaMetadataV1:fieldName=random1'`),
118-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random2" Nullable(String), COMMENT COLUMN "random2" 'quesmaMetadataV1:fieldName=random2'`),
119-
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String), COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`),
120+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random1" Array(String)`),
121+
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "random1" 'quesmaMetadataV1:fieldName=random1'`),
122+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random2" Nullable(String)`),
123+
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "random2" 'quesmaMetadataV1:fieldName=random2'`),
124+
EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String)`),
125+
EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`),
120126
EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","random1":["debug"],"random2":"random-string","severity":"frontend"}`),
121127
},
122128
}

‎platform/ingest/processor.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -382,11 +382,13 @@ func (ip *IngestProcessor) generateNewColumns(
382382
maybeOnClusterClause = " ON CLUSTER " + strconv.Quote(table.ClusterName)
383383
}
384384

385-
alterTable := fmt.Sprintf(`ALTER TABLE "%s"%s ADD COLUMN IF NOT EXISTS "%s" %s, COMMENT COLUMN "%s" '%s'`,
386-
table.Name, maybeOnClusterClause, attrKeys[i], columnType, attrKeys[i], comment)
385+
alterTable := fmt.Sprintf("ALTER TABLE \"%s\"%s ADD COLUMN IF NOT EXISTS \"%s\" %s", table.Name, maybeOnClusterClause, attrKeys[i], columnType)
387386
newColumns[attrKeys[i]] = &chLib.Column{Name: attrKeys[i], Type: chLib.NewBaseType(attrTypes[i]), Modifiers: modifiers, Comment: comment}
388387
alterCmd = append(alterCmd, alterTable)
389388

389+
alterColumn := fmt.Sprintf("ALTER TABLE \"%s\"%s COMMENT COLUMN \"%s\" '%s'", table.Name, maybeOnClusterClause, attrKeys[i], comment)
390+
alterCmd = append(alterCmd, alterColumn)
391+
390392
deleteIndexes = append(deleteIndexes, i)
391393
}
392394

0 commit comments

Comments
 (0)