Regression fix for CREATE TABLE ... ON CLUSTER

platform/database_common/log_manager_test.go

@@ -8,7 +8,6 @@ import (
 	schema2 "github.com/QuesmaOrg/quesma/platform/schema"
 	"github.com/QuesmaOrg/quesma/platform/types"
 	"github.com/QuesmaOrg/quesma/platform/util"
-	"strings"
 	"sync/atomic"
 	"testing"
 
@@ -465,161 +464,6 @@ func TestJsonConvertingBoolToStringAttr(t *testing.T) {
 	}
 }
 
-// Doesn't test for 100% equality, as map iteration order isn't deterministic, but should definitely be good enough.
-func TestCreateTableString_1(t *testing.T) {
-	table := Table{
-		Name: "/_bulk?refresh=false&_source_includes=originId&require_alias=true_16",
-		Cols: map[string]*Column{
-			"doc": {
-				Name: "doc",
-				Type: MultiValueType{
-					Name: "Tuple",
-					Cols: []*Column{
-						{
-							Name: "Tuple",
-							Type: MultiValueType{
-								Name: "Tuple",
-								Cols: []*Column{
-									{
-										Name: "runAt",
-										Type: NewBaseType("DateTime64"),
-									},
-									{
-										Name: "startedAt",
-										Type: NewBaseType("DateTime64"),
-									},
-									{
-										Name: "Tuple",
-										Type: NewBaseType("String"),
-									},
-									{
-										Name: "status",
-										Type: NewBaseType("String"),
-									},
-								},
-							},
-						},
-						{
-							Name: "updated_at",
-							Type: NewBaseType("DateTime64"),
-						},
-					},
-				},
-			},
-			"@timestamp": {
-				Name: "@timestamp",
-				Type: NewBaseType("DateTime64"),
-			},
-		},
-		Config: &ChTableConfig{
-			HasTimestamp:         true,
-			TimestampDefaultsNow: true,
-			Engine:               "MergeTree",
-			OrderBy:              "(@timestamp)",
-			PrimaryKey:           "",
-			Ttl:                  "",
-			Attributes: []Attribute{
-				NewDefaultStringAttribute(),
-			},
-			CastUnsupportedAttrValueTypesToString: false,
-			PreferCastingToOthers:                 false,
-		},
-	}
-	expectedRows := []string{
-		`CREATE TABLE IF NOT EXISTS "/_bulk?refresh=false&_source_includes=originId&require_alias=true_16" (`,
-		`"doc" Tuple`,
-		`(`,
-		`"Tuple" Tuple`,
-		`(`,
-		`"runAt" DateTime64,`,
-		`"startedAt" DateTime64,`,
-		`"Tuple" String,`,
-		`"status" String`,
-		`),`,
-		`"updated_at" DateTime64`,
-		`),`,
-		`"@timestamp" DateTime64,`,
-		`"attributes_values" Map(String,String),`,
-		`"attributes_metadata" Map(String,String)`,
-		`)`,
-		`ENGINE = MergeTree`,
-		`ORDER BY (@timestamp)`,
-		"",
-	}
-	createTableString := table.CreateTableString()
-	for _, row := range strings.Split(createTableString, "\n") {
-		assert.Contains(t, expectedRows, strings.TrimSpace(row))
-	}
-}
-
-// Doesn't test for 100% equality, as map iteration order isn't deterministic, but should definitely be good enough.
-func TestCreateTableString_NewDateTypes(t *testing.T) {
-	table := Table{
-		Name: "abc",
-		Cols: map[string]*Column{
-			"low_card_string": {
-				Name:    "low_card_string",
-				Type:    NewBaseType("LowCardinality(String)"),
-				Comment: "some comment 1",
-			},
-			"uuid": {
-				Name: "uuid",
-				Type: NewBaseType("UUID"),
-			},
-			"int32": {
-				Name:    "int32",
-				Type:    NewBaseType("Int32"),
-				Comment: "some comment 2",
-			},
-			"epoch_time": {
-				Name:      "epoch_time",
-				Type:      NewBaseType("DateTime('Asia/Kolkata')"),
-				Modifiers: "CODEC(DoubleDelta, LZ4)",
-			},
-			"estimated_connection_speedinkbps": {
-				Name:      "estimated_connection_speedinkbps",
-				Type:      NewBaseType("Float64"),
-				Modifiers: "CODEC(DoubleDelta, LZ4)",
-			},
-		},
-		Config: &ChTableConfig{
-			HasTimestamp:         true,
-			TimestampDefaultsNow: true,
-			Engine:               "MergeTree",
-			OrderBy:              "(@timestamp)",
-			PrimaryKey:           "",
-			Ttl:                  "",
-			Attributes: []Attribute{
-				NewDefaultInt64Attribute(),
-			},
-			CastUnsupportedAttrValueTypesToString: true,
-			PreferCastingToOthers:                 true,
-		},
-	}
-	expectedRows := []string{
-		`CREATE TABLE IF NOT EXISTS "abc" (`,
-		`"int32" Int32 COMMENT 'some comment 2',`,
-		`"low_card_string" LowCardinality(String) COMMENT 'some comment 1',`,
-		`"uuid" UUID,`,
-		`"others" JSON,`,
-		`"attributes_int64_key" Array(String),`,
-		`"attributes_int64_value" Array(Int64),`,
-		`"attributes_values" Map(String,String),`,
-		`"attributes_metadata" Map(String,String)`,
-		`"@timestamp" DateTime64(3) DEFAULT now64(),`,
-		`"epoch_time" DateTime('Asia/Kolkata') CODEC(DoubleDelta, LZ4),`,
-		`"estimated_connection_speedinkbps" Float64 CODEC(DoubleDelta, LZ4),`,
-		`ENGINE = MergeTree`,
-		`)`,
-		`ORDER BY (@timestamp)`,
-		"",
-	}
-	createTableString := table.CreateTableString()
-	for _, row := range strings.Split(createTableString, "\n") {
-		assert.Contains(t, expectedRows, strings.TrimSpace(row))
-	}
-}
-
 func TestLogManager_GetTable(t *testing.T) {
 	tests := []struct {
 		name             string

platform/database_common/table.go

@@ -8,7 +8,6 @@ import (
 	"github.com/QuesmaOrg/quesma/platform/config"
 	"github.com/QuesmaOrg/quesma/platform/logger"
 	"github.com/QuesmaOrg/quesma/platform/model"
-	"github.com/QuesmaOrg/quesma/platform/util"
 	"strconv"
 	"strings"
 )
@@ -30,48 +29,6 @@ type Table struct {
 	ExistsOnAllNodes bool
 }
 
-func (t *Table) createTableOurFieldsString() []string {
-	rows := make([]string, 0)
-	if t.Config.HasTimestamp {
-		_, ok := t.Cols[timestampFieldName]
-		if !ok {
-			defaultStr := ""
-			if t.Config.TimestampDefaultsNow {
-				defaultStr = " DEFAULT now64()"
-			}
-			rows = append(rows, fmt.Sprintf("%s\"%s\" DateTime64(3)%s", util.Indent(1), timestampFieldName, defaultStr))
-		}
-	}
-	if len(t.Config.Attributes) > 0 {
-		for _, a := range t.Config.Attributes {
-			_, ok := t.Cols[a.MapValueName]
-			if !ok {
-				rows = append(rows, fmt.Sprintf("%s\"%s\" Map(String,String)", util.Indent(1), a.MapValueName))
-			}
-			_, ok = t.Cols[a.MapMetadataName]
-			if !ok {
-				rows = append(rows, fmt.Sprintf("%s\"%s\" Map(String,String)", util.Indent(1), a.MapMetadataName))
-			}
-
-		}
-	}
-	return rows
-}
-
-func (t *Table) CreateTableString() string {
-	var onClusterClause string
-	if t.ClusterName != "" {
-		onClusterClause = " ON CLUSTER " + strconv.Quote(t.ClusterName)
-	}
-	s := "CREATE TABLE IF NOT EXISTS " + t.FullTableName() + onClusterClause + " (\n"
-	rows := make([]string, 0)
-	for _, col := range t.Cols {
-		rows = append(rows, col.createTableString(1))
-	}
-	rows = append(rows, t.createTableOurFieldsString()...)
-	return s + strings.Join(rows, ",\n") + "\n)\n" + t.Config.CreateTablePostFieldsString()
-}
-
 // FullTableName returns full table name with database name if it's not empty.
 // Format: ["database".]"table" as it seems to work for all cases in Clickhouse.
 // Use this in Clickhouse queries, e.g. in FROM clause.

platform/ingest/ast.go

@@ -53,7 +53,7 @@ func (ct CreateTableStatement) ToSQL() string {
 	var b strings.Builder
 
 	if ct.Cluster != "" {
-		b.WriteString(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%s" ON CLUSTER "%s"`+" \n(\n\n", ct.Name, ct.Cluster))
+		b.WriteString(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%s" ON CLUSTER "%s"`+" \n", ct.Name, ct.Cluster))
 	} else {
 		b.WriteString(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%s"`, ct.Name))
 	}

platform/ingest/ast_test.go

@@ -0,0 +1,67 @@
+// Copyright Quesma, licensed under the Elastic License 2.0.
+// SPDX-License-Identifier: Elastic-2.0
+
+package ingest
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestCreateTableStatement_ToSQL(t *testing.T) {
+	tests := []struct {
+		name     string
+		stmt     CreateTableStatement
+		expected string
+	}{
+		{
+			name: "basic table",
+			stmt: CreateTableStatement{
+				Name: "my_table",
+				Columns: []ColumnStatement{
+					{ColumnName: "id", ColumnType: "Int64"},
+					{ColumnName: "@timestamp", ColumnType: "DateTime64(3)", AdditionalMetadata: "DEFAULT now64()"},
+					{ColumnName: "name", ColumnType: "String", Comment: "user name"},
+				},
+				Comment:    "created by Quesma",
+				PostClause: "ENGINE = MergeTree() ORDER BY (\"@timestamp\")",
+			},
+			expected: `CREATE TABLE IF NOT EXISTS "my_table" 
+(
+
+	"id" Int64,
+	"@timestamp" DateTime64(3) DEFAULT now64(),
+	"name" String COMMENT 'user name'
+)
+ENGINE = MergeTree() ORDER BY ("@timestamp")
+COMMENT 'created by Quesma'`,
+		},
+		{
+			name: "basic table with cluster",
+			stmt: CreateTableStatement{
+				Name:    "my_table",
+				Cluster: "quesma_cluster",
+				Columns: []ColumnStatement{
+					{ColumnName: "id", ColumnType: "Int64"},
+				},
+				Comment:    "created by Quesma",
+				PostClause: "ENGINE = MergeTree() ORDER BY (\"id\")",
+			},
+			expected: `CREATE TABLE IF NOT EXISTS "my_table" ON CLUSTER "quesma_cluster" 
+
+(
+
+	"id" Int64
+)
+ENGINE = MergeTree() ORDER BY ("id")
+COMMENT 'created by Quesma'`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.stmt.ToSQL()
+			assert.Equal(t, tt.expected, got)
+		})
+	}
+}