diff --git a/platform/ingest/alter_table_test.go b/platform/ingest/alter_table_test.go index 584f491c1..75221289f 100644 --- a/platform/ingest/alter_table_test.go +++ b/platform/ingest/alter_table_test.go @@ -36,8 +36,8 @@ func TestAlterTable(t *testing.T) { "{\"Test1\":1,\"Test2\":2}", } alters := []string{ - "ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test1\" Nullable(Int64), COMMENT COLUMN \"Test1\" 'quesmaMetadataV1:fieldName=Test1'", - "ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test2\" Nullable(Int64), COMMENT COLUMN \"Test2\" 'quesmaMetadataV1:fieldName=Test2'", + "ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test1\" Nullable(Int64)", + "ALTER TABLE \"tableName\" ADD COLUMN IF NOT EXISTS \"Test2\" Nullable(Int64)", } columns := []string{"Test1", "Test2"} table := &clickhouse.Table{ diff --git a/platform/ingest/common_table_test.go b/platform/ingest/common_table_test.go index 34170767c..99b8f2d47 100644 --- a/platform/ingest/common_table_test.go +++ b/platform/ingest/common_table_test.go @@ -35,7 +35,8 @@ func TestIngestToCommonTable(t *testing.T) { {"foo": "bar"}, }, expectedStatements: []string{ - `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`, + `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`, + `ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`, `INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}`, }, virtualTableColumns: []string{"@timestamp", "foo"}, @@ -47,7 +48,8 @@ func TestIngestToCommonTable(t *testing.T) { {"foo": "baz"}, }, expectedStatements: []string{ - `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`, + `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`, + `ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`, `INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}`, }, virtualTableColumns: []string{"@timestamp", "foo"}, @@ -60,8 +62,10 @@ func TestIngestToCommonTable(t *testing.T) { {"foo": "1", "baz": "qux"}, }, expectedStatements: []string{ - `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`, - `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String), COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`, + `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`, + `ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`, + `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String)`, + `ALTER TABLE "quesma_common_table" COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`, `INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}, {"__quesma_index_name":"test_index","baz":"qux","foo":"1"} `, }, @@ -78,8 +82,10 @@ func TestIngestToCommonTable(t *testing.T) { {"foo": "1", "baz": "qux"}, }, expectedStatements: []string{ - `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String), COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`, - `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String), COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`, + `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "foo" Nullable(String)`, + `ALTER TABLE "quesma_common_table" COMMENT COLUMN "foo" 'quesmaMetadataV1:fieldName=foo'`, + `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "baz" Nullable(String)`, + `ALTER TABLE "quesma_common_table" COMMENT COLUMN "baz" 'quesmaMetadataV1:fieldName=baz'`, `INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","foo":"bar"}, {"__quesma_index_name":"test_index","foo":"baz"}, {"__quesma_index_name":"test_index","baz":"qux","foo":"1"} `, }, @@ -107,7 +113,8 @@ func TestIngestToCommonTable(t *testing.T) { {"a": "bar", "b": "baz"}, }, expectedStatements: []string{ - `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "b" Nullable(String), COMMENT COLUMN "b" 'quesmaMetadataV1:fieldName=b'`, + `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "b" Nullable(String)`, + `ALTER TABLE "quesma_common_table" COMMENT COLUMN "b" 'quesmaMetadataV1:fieldName=b'`, `INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","a":"bar","b":"baz"}`, }, @@ -120,7 +127,8 @@ func TestIngestToCommonTable(t *testing.T) { {"a.b": "c"}, }, expectedStatements: []string{ - `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "a_b" Nullable(String), COMMENT COLUMN "a_b" 'quesmaMetadataV1:fieldName=a.b'`, + `ALTER TABLE "quesma_common_table" ADD COLUMN IF NOT EXISTS "a_b" Nullable(String)`, + `ALTER TABLE "quesma_common_table" COMMENT COLUMN "a_b" 'quesmaMetadataV1:fieldName=a.b'`, `INSERT INTO "quesma_common_table" FORMAT JSONEachRow {"__quesma_index_name":"test_index","a_b":"c"}`, }, diff --git a/platform/ingest/insert_test.go b/platform/ingest/insert_test.go index a42efb4ef..7f31a929c 100644 --- a/platform/ingest/insert_test.go +++ b/platform/ingest/insert_test.go @@ -102,11 +102,14 @@ var configs = []*clickhouse.ChTableConfig{ var expectedInserts = [][]string{ []string{EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","service_name":"frontend","severity":"debug","source":"rhel"}`)}, []string{ - EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "service_name" Nullable(String), COMMENT COLUMN "service_name" 'quesmaMetadataV1:fieldName=service.name'`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "service_name" Nullable(String)`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "service_name" 'quesmaMetadataV1:fieldName=service.name`), - EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String), COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String)`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`), - EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "source" Nullable(String), COMMENT COLUMN "source" 'quesmaMetadataV1:fieldName=source'`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "source" Nullable(String)`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "source" 'quesmaMetadataV1:fieldName=source'`), EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","service_name":"frontend","severity":"debug","source":"rhel"}`), }, @@ -114,9 +117,12 @@ var expectedInserts = [][]string{ EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","random1":["debug"],"random2":"random-string","severity":"frontend"}`), }, []string{ - EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random1" Array(String), COMMENT COLUMN "random1" 'quesmaMetadataV1:fieldName=random1'`), - EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random2" Nullable(String), COMMENT COLUMN "random2" 'quesmaMetadataV1:fieldName=random2'`), - EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String), COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random1" Array(String)`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "random1" 'quesmaMetadataV1:fieldName=random1'`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "random2" Nullable(String)`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "random2" 'quesmaMetadataV1:fieldName=random2'`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" ADD COLUMN IF NOT EXISTS "severity" Nullable(String)`), + EscapeBrackets(`ALTER TABLE "` + tableName + `" COMMENT COLUMN "severity" 'quesmaMetadataV1:fieldName=severity'`), EscapeBrackets(`INSERT INTO "` + tableName + `" FORMAT JSONEachRow {"@timestamp":"2024-01-27T16:11:19.94Z","host_name":"hermes","message":"User password reset failed","random1":["debug"],"random2":"random-string","severity":"frontend"}`), }, } diff --git a/platform/ingest/processor.go b/platform/ingest/processor.go index f7a936249..805e5bde6 100644 --- a/platform/ingest/processor.go +++ b/platform/ingest/processor.go @@ -382,11 +382,13 @@ func (ip *IngestProcessor) generateNewColumns( maybeOnClusterClause = " ON CLUSTER " + strconv.Quote(table.ClusterName) } - alterTable := fmt.Sprintf(`ALTER TABLE "%s"%s ADD COLUMN IF NOT EXISTS "%s" %s, COMMENT COLUMN "%s" '%s'`, - table.Name, maybeOnClusterClause, attrKeys[i], columnType, attrKeys[i], comment) + alterTable := fmt.Sprintf("ALTER TABLE \"%s\"%s ADD COLUMN IF NOT EXISTS \"%s\" %s", table.Name, maybeOnClusterClause, attrKeys[i], columnType) newColumns[attrKeys[i]] = &chLib.Column{Name: attrKeys[i], Type: chLib.NewBaseType(attrTypes[i]), Modifiers: modifiers, Comment: comment} alterCmd = append(alterCmd, alterTable) + alterColumn := fmt.Sprintf("ALTER TABLE \"%s\"%s COMMENT COLUMN \"%s\" '%s'", table.Name, maybeOnClusterClause, attrKeys[i], comment) + alterCmd = append(alterCmd, alterColumn) + deleteIndexes = append(deleteIndexes, i) }