From 5561070df86e30a16867bf4b66c2bebcbac23a21 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Wed, 4 Jun 2025 16:49:06 +0200 Subject: [PATCH 01/12] Remove unused indexes --- platform/clickhouse/clickhouse_test.go | 16 +--------------- platform/clickhouse/table.go | 4 ---- platform/ingest/processor_test.go | 16 +--------------- 3 files changed, 2 insertions(+), 34 deletions(-) diff --git a/platform/clickhouse/clickhouse_test.go b/platform/clickhouse/clickhouse_test.go index e504a4fb3..b5d8e40e2 100644 --- a/platform/clickhouse/clickhouse_test.go +++ b/platform/clickhouse/clickhouse_test.go @@ -519,17 +519,11 @@ func TestCreateTableString_1(t *testing.T) { PrimaryKey: "", Ttl: "", Attributes: []Attribute{ - NewDefaultInt64Attribute(), NewDefaultStringAttribute(), - NewDefaultBoolAttribute(), }, CastUnsupportedAttrValueTypesToString: false, PreferCastingToOthers: false, }, - Indexes: []IndexStatement{ - GetIndexStatement("body"), - GetIndexStatement("severity"), - }, } expectedRows := []string{ `CREATE TABLE IF NOT EXISTS "/_bulk?refresh=false&_source_includes=originId&require_alias=true_16" (`, @@ -545,16 +539,8 @@ func TestCreateTableString_1(t *testing.T) { `"updated_at" DateTime64`, `),`, `"@timestamp" DateTime64,`, - `"attributes_int64_key" Array(String),`, - `"attributes_int64_value" Array(Int64),`, - `"attributes_string_key" Array(String),`, - `"attributes_string_value" Array(String),`, - `"attributes_bool_key" Array(String),`, - `"attributes_bool_value" Array(Bool),`, `"attributes_values" Map(String,String),`, - `"attributes_metadata" Map(String,String),`, - `INDEX body_idx body TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4,`, - `INDEX severity_idx severity TYPE set(25) GRANULARITY 4`, + `"attributes_metadata" Map(String,String)`, `)`, `ENGINE = MergeTree`, `ORDER BY (@timestamp)`, diff --git a/platform/clickhouse/table.go b/platform/clickhouse/table.go index 31e7d9c17..947ffb5ad 100644 --- a/platform/clickhouse/table.go +++ b/platform/clickhouse/table.go @@ -19,7 +19,6 @@ type Table struct { ClusterName string `default:""` Cols map[string]*Column Config *ChTableConfig - Indexes []IndexStatement aliases map[string]string //deprecated // TODO: we should use aliases directly from configuration, not store them here Comment string // this human-readable comment @@ -70,9 +69,6 @@ func (t *Table) CreateTableString() string { rows = append(rows, col.createTableString(1)) } rows = append(rows, t.createTableOurFieldsString()...) - for _, index := range t.Indexes { - rows = append(rows, util.Indent(1)+index.Statement()) - } return s + strings.Join(rows, ",\n") + "\n)\n" + t.Config.CreateTablePostFieldsString() } diff --git a/platform/ingest/processor_test.go b/platform/ingest/processor_test.go index de7c3efbf..8bb6d2826 100644 --- a/platform/ingest/processor_test.go +++ b/platform/ingest/processor_test.go @@ -655,17 +655,11 @@ func TestCreateTableString_1(t *testing.T) { PrimaryKey: "", Ttl: "", Attributes: []clickhouse.Attribute{ - clickhouse.NewDefaultInt64Attribute(), clickhouse.NewDefaultStringAttribute(), - clickhouse.NewDefaultBoolAttribute(), }, CastUnsupportedAttrValueTypesToString: false, PreferCastingToOthers: false, }, - Indexes: []clickhouse.IndexStatement{ - clickhouse.GetIndexStatement("body"), - clickhouse.GetIndexStatement("severity"), - }, } expectedRows := []string{ `CREATE TABLE IF NOT EXISTS "/_bulk?refresh=false&_source_includes=originId&require_alias=true_16" (`, @@ -681,16 +675,8 @@ func TestCreateTableString_1(t *testing.T) { `"updated_at" DateTime64`, `),`, `"@timestamp" DateTime64,`, - `"attributes_int64_key" Array(String),`, - `"attributes_int64_value" Array(Int64),`, - `"attributes_string_key" Array(String),`, - `"attributes_string_value" Array(String),`, - `"attributes_bool_key" Array(String),`, - `"attributes_bool_value" Array(Bool),`, `"attributes_values" Map(String,String),`, - `"attributes_metadata" Map(String,String),`, - `INDEX body_idx body TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4,`, - `INDEX severity_idx severity TYPE set(25) GRANULARITY 4`, + `"attributes_metadata" Map(String,String)`, `)`, `ENGINE = MergeTree`, `ORDER BY (@timestamp)`, From 7f1cba6b5cd3175f5f47181a2739fe23cbe2d8b2 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 10:59:47 +0200 Subject: [PATCH 02/12] Remove NewTable from tests --- cmd/v2_test_objects.go | 18 +- platform/clickhouse/schema_test.go | 16 +- .../schema_transformer_test.go | 13 +- platform/ingest/parserCreateTable_test.go | 387 ------------------ 4 files changed, 28 insertions(+), 406 deletions(-) delete mode 100644 platform/ingest/parserCreateTable_test.go diff --git a/cmd/v2_test_objects.go b/cmd/v2_test_objects.go index dd44938d5..2e172c79b 100644 --- a/cmd/v2_test_objects.go +++ b/cmd/v2_test_objects.go @@ -388,17 +388,19 @@ func (p *QueryTransformationPipeline) ParseQuery(message any) (*model.ExecutionP // TODO this is a hack to create a table for the query // Why parser needs a table? tableName := "test_table" - table, err := clickhouse.NewTable(`CREATE TABLE `+tableName+` - ( "message" String, "@timestamp" DateTime64(3, 'UTC'), "attributes_values" Map(String,String)) - ENGINE = Memory`, - clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), - ) - if err != nil { - return nil, err + table := clickhouse.Table{ + Name: tableName, + DatabaseName: "default", + Cols: map[string]*clickhouse.Column{ + "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, + "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64(3, 'UTC')")}, + "attributes_values": {Name: "attributes_values", Type: clickhouse.NewBaseType("Map(String,String)")}, + }, + Config: clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), } cw := elastic_query_dsl.ClickhouseQueryTranslator{ Ctx: req.OriginalRequest.Context(), - Table: table, + Table: &table, } plan, err := cw.ParseQuery(query) if err != nil { diff --git a/platform/clickhouse/schema_test.go b/platform/clickhouse/schema_test.go index e77223bd6..751ab7a09 100644 --- a/platform/clickhouse/schema_test.go +++ b/platform/clickhouse/schema_test.go @@ -10,12 +10,16 @@ import ( func TestGetDateTimeType(t *testing.T) { ctx := context.Background() - table, err := NewTable(`CREATE TABLE table ( - "timestamp1" DateTime, - "timestamp2" DateTime('UTC'), - "timestamp64_1" DateTime64, - "timestamp64_2" DateTime64(3, 'UTC') ) ENGINE = Memory`, NewChTableConfigTimestampStringAttr()) - assert.NoError(t, err) + table := Table{ + Name: "table", + Cols: map[string]*Column{ + "timestamp1": {Name: "timestamp1", Type: NewBaseType("DateTime")}, + "timestamp2": {Name: "timestamp2", Type: NewBaseType("DateTime('UTC')")}, + "timestamp64_1": {Name: "timestamp64_1", Type: NewBaseType("DateTime64")}, + "timestamp64_2": {Name: "timestamp64_2", Type: NewBaseType("DateTime64(3, 'UTC')")}, + }, + Config: NewChTableConfigTimestampStringAttr(), + } assert.Equal(t, DateTime, table.GetDateTimeType(ctx, "timestamp1", true)) assert.Equal(t, DateTime, table.GetDateTimeType(ctx, "timestamp2", true)) assert.Equal(t, DateTime64, table.GetDateTimeType(ctx, "timestamp64_1", true)) diff --git a/platform/frontend_connectors/schema_transformer_test.go b/platform/frontend_connectors/schema_transformer_test.go index 821d8927e..a23df6a9f 100644 --- a/platform/frontend_connectors/schema_transformer_test.go +++ b/platform/frontend_connectors/schema_transformer_test.go @@ -2121,11 +2121,14 @@ func Test_acceptIntsAsTimestamps(t *testing.T) { tableMap := clickhouse.NewTableMap() // timestampInt is datetime in schema (and Quesma config), UInt64 in Clickhouse - tab, _ := clickhouse.NewTable(` - CREATE TABLE table ( - "timestampInt" UInt64 - ) ENGINE = Memory`, clickhouse.NewChTableConfigTimestampStringAttr()) - tableMap.Store("test", tab) + tab := clickhouse.Table{ + Name: "test", + Config: clickhouse.NewChTableConfigTimestampStringAttr(), + Cols: map[string]*clickhouse.Column{ + "timestampInt": {Name: "timestampInt", Type: clickhouse.NewBaseType("UInt32")}, + }, + } + tableMap.Store("test", &tab) td := clickhouse.NewEmptyTableDiscovery() td.TableMap = tableMap diff --git a/platform/ingest/parserCreateTable_test.go b/platform/ingest/parserCreateTable_test.go deleted file mode 100644 index 271447ede..000000000 --- a/platform/ingest/parserCreateTable_test.go +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright Quesma, licensed under the Elastic License 2.0. -// SPDX-License-Identifier: Elastic-2.0 -package ingest - -import ( - "github.com/QuesmaOrg/quesma/platform/clickhouse" - "github.com/QuesmaOrg/quesma/platform/util" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParseSignozSchema_1(t *testing.T) { - q := `CREATE TABLE signoz_logs - ( - "timestamp" UInt64 CODEC(DoubleDelta, LZ4), - "observed_timestamp" UInt64 CODEC(DoubleDelta, LZ4), - "id" String CODEC(ZSTD(1)), - "trace_id" String CODEC(ZSTD(1)), - "span_id" String CODEC(ZSTD(1)), - "trace_flags" UInt32, - "severity_text" LowCardinality(String) CODEC(ZSTD(1)), - "severity_number" UInt8, - "body" String CODEC(ZSTD(2)), - "resources_string_key" Array(String) CODEC(ZSTD(1)), - "resources_string_value" Array(String) CODEC(ZSTD(1)), - "attributes_string_key" Array(String) CODEC(ZSTD(1)), - "attributes_string_value" Array(String) CODEC(ZSTD(1)), - "attributes_int64_key" Array(String) CODEC(ZSTD(1)), - "attributes_int64_value" Array(Int64) CODEC(ZSTD(1)), - "attributes_float64_key" Array(String) CODEC(ZSTD(1)), - "attributes_float64_value" Array(Float64) CODEC(ZSTD(1)), - "attributes_bool_key" Array(String) CODEC(ZSTD(1)), - "attributes_bool_value" Array(Bool) CODEC(ZSTD(1)), - )` - fieldNames := []string{"timestamp", "observed_timestamp", "id", "trace_id", "span_id", "trace_flags", "severity_text", "severity_number", "body", "resources_string_key", "resources_string_value", "attributes_string_key", "attributes_string_value", "attributes_int64_key", "attributes_int64_value", "attributes_float64_key", "attributes_float64_value", "attributes_bool_key", "attributes_bool_value"} - table, err := clickhouse.NewTable(q, nil) - assert.NoError(t, err) - assert.Equal(t, len(fieldNames), len(table.Cols)) - for _, fieldName := range fieldNames { - assert.Contains(t, table.Cols, fieldName) - } -} - -func TestParseSignozSchema_2(t *testing.T) { - // we test here using both "name" and `name` for column names - q := `CREATE TABLE IF NOT EXISTS db.signoz_logs ON CLUSTER cluster - ( - ` + "`" + "@timestamp" + "` " + `UInt64 CODEC(DoubleDelta, LZ4), - "observed_timestamp" UInt64 CODEC(DoubleDelta, LZ4), - "timestampDT64_1" DateTime64(6, 'UTC') DEFAULT toDateTime64(timestamp, 6, 'UTC') CODEC(DoubleDelta, LZ4), - "timestampDT64_2" DateTime64(6, 'UTC') DEFAULT now() + toDateTime64(timestamp, 6, 'UTC'), - "timestampDT64_3" DateTime64(6, 'UTC'), - "id" String NOT NULL CODEC(ZSTD(1)), - "trace_id" String DEFAULT "hehe" CODEC(ZSTD(1)), - "span_id" String NULL CODEC(ZSTD(1)), - "trace_flags" Uint32 NOT NULL DEFAULT 5, - "severity_text" LowCardinality(String) CODEC(ZSTD(1)), - "severity_number" UInt8, - "body" String CODEC(ZSTD(2)), - "resources_string_key" Array(String) CODEC(ZSTD(1)), - "resources_string_value" Array(String) CODEC(ZSTD(1)) TTL 0, - "attributes_string_key" Array(String) CODEC(ZSTD(1)), - "attributes_string_value" Array(String) CODEC(ZSTD(1)), - "attributes_int64_key" Array(String) CODEC(ZSTD(1)), - "attributes_int64_value" Array(Int64) CODEC(ZSTD(1)) TTL 5555, - "attributes_float64_key" Array(String) CODEC(ZSTD(1)), - "attributes_float64_value" Array(Float64) CODEC(ZSTD(1)), - "attributes_bool_key" Array(String) CODEC(ZSTD(1)) TTL 10 + 50 * 80 + now(), - "attributes_bool_value" Array(Bool) CODEC(ZSTD(1)), - "tuple1" Tuple(a String, b String, c Tuple(c String, d Uint128)) CODEC(ZSTD(1)), - )` - fieldNames := []string{"@timestamp", "observed_timestamp", "timestampDT64_1", "timestampDT64_2", "timestampDT64_3", "id", "trace_id", "span_id", "trace_flags", "severity_text", "severity_number", "body", "resources_string_key", "resources_string_value", "attributes_string_key", "attributes_string_value", "attributes_int64_key", "attributes_int64_value", "attributes_float64_key", "attributes_float64_value", "attributes_bool_key", "attributes_bool_value", "tuple1"} - table, err := clickhouse.NewTable(q, nil) - assert.NoError(t, err) - assert.Equal(t, len(fieldNames), len(table.Cols)) - for _, fieldName := range fieldNames { - assert.Contains(t, table.Cols, fieldName) - } - assert.Equal(t, "db", table.DatabaseName) - assert.Equal(t, "cluster", table.ClusterName) -} - -func TestParseQuotedTablename(t *testing.T) { - q := `CREATE TABLE IF NOT EXISTS "logs-generic-default" - ( - "source" String, - "host.name" String, - "message" String, - "service.name" String, - "severity" String - ) - ENGINE = MergeTree - ORDER BY (timestamp)` - fieldNames := []string{"source", "host.name", "message", "service.name", "severity"} - table, err := clickhouse.NewTable(q, nil) - assert.NoError(t, err) - assert.Equal(t, len(fieldNames), len(table.Cols)) - for _, fieldName := range fieldNames { - assert.Contains(t, table.Cols, fieldName) - } -} - -func TestParseNonLetterNames(t *testing.T) { - q := `CREATE TABLE IF NOT EXISTS "/_monitoring/bulk?system_id=kibana&system_api_version=7&interval=10000ms_1" - ( - "index" Tuple - ( - "_type" String - ) - ) - ENGINE = MergeTree - ORDER BY (timestamp)` - fieldNames := []string{"index"} - table, err := clickhouse.NewTable(q, nil) - assert.NoError(t, err) - assert.Equal(t, len(fieldNames), len(table.Cols)) - for _, fieldName := range fieldNames { - assert.Contains(t, table.Cols, fieldName) - } -} - -func TestParseCreateSampleDataEcommerce(t *testing.T) { - q := `CREATE TABLE IF NOT EXISTS "kibana_sample_data_ecommerce" ON CLUSTER "quesma_cluster" -( - "@timestamp" DateTime64(3) DEFAULT now64(), - "attributes_values" Map(String,String), - "attributes_metadata" Map(String,String), - - - "taxful_total_price" Nullable(Float64) COMMENT 'quesmaMetadataV1:fieldName=taxful_total_price', - "sku" Array(String) COMMENT 'quesmaMetadataV1:fieldName=sku', - "taxless_total_price" Nullable(Float64) COMMENT 'quesmaMetadataV1:fieldName=taxless_total_price', - "total_unique_products" Nullable(Int64) COMMENT 'quesmaMetadataV1:fieldName=total_unique_products', - "geoip_region_name" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=geoip.region_name', - "category" Array(String) COMMENT 'quesmaMetadataV1:fieldName=category', - "products_created_on" Array(DateTime64) COMMENT 'quesmaMetadataV1:fieldName=products.created_on', - "products_taxful_price" Array(Float64) COMMENT 'quesmaMetadataV1:fieldName=products.taxful_price', - "user" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=user', - "currency" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=currency', - "day_of_week" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=day_of_week', - "geoip_location_lat" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=geoip.location.lat', - "geoip_city_name" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=geoip.city_name', - "products__id" Array(String) COMMENT 'quesmaMetadataV1:fieldName=products._id', - "products_discount_amount" Array(Int64) COMMENT 'quesmaMetadataV1:fieldName=products.discount_amount', - "customer_last_name" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=customer_last_name', - "total_quantity" Nullable(Int64) COMMENT 'quesmaMetadataV1:fieldName=total_quantity', - "geoip_continent_name" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=geoip.continent_name', - "products_price" Array(Float64) COMMENT 'quesmaMetadataV1:fieldName=products.price', - "products_sku" Array(String) COMMENT 'quesmaMetadataV1:fieldName=products.sku', - "products_discount_percentage" Array(Int64) COMMENT 'quesmaMetadataV1:fieldName=products.discount_percentage', - "type" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=type', - "customer_full_name" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=customer_full_name', - "products_min_price" Array(Float64) COMMENT 'quesmaMetadataV1:fieldName=products.min_price', - "products_unit_discount_amount" Array(Int64) COMMENT 'quesmaMetadataV1:fieldName=products.unit_discount_amount', - "products_manufacturer" Array(String) COMMENT 'quesmaMetadataV1:fieldName=products.manufacturer', - "products_base_unit_price" Array(Float64) COMMENT 'quesmaMetadataV1:fieldName=products.base_unit_price', - "day_of_week_i" Nullable(Int64) COMMENT 'quesmaMetadataV1:fieldName=day_of_week_i', - "manufacturer" Array(String) COMMENT 'quesmaMetadataV1:fieldName=manufacturer', - "customer_first_name" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=customer_first_name', - "products_product_id" Array(Int64) COMMENT 'quesmaMetadataV1:fieldName=products.product_id', - "customer_gender" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=customer_gender', - "email" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=email', - "order_id" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=order_id', - "customer_phone" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=customer_phone', - "order_date" Nullable(DateTime64) COMMENT 'quesmaMetadataV1:fieldName=order_date', - "geoip_location_lon" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=geoip.location.lon', - "products_base_price" Array(Float64) COMMENT 'quesmaMetadataV1:fieldName=products.base_price', - "products_category" Array(String) COMMENT 'quesmaMetadataV1:fieldName=products.category', - "products_tax_amount" Array(Int64) COMMENT 'quesmaMetadataV1:fieldName=products.tax_amount', - "products_product_name" Array(String) COMMENT 'quesmaMetadataV1:fieldName=products.product_name', - "customer_id" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=customer_id', - "event_dataset" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=event.dataset', - "geoip_country_iso_code" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=geoip.country_iso_code', - "products_taxless_price" Array(Float64) COMMENT 'quesmaMetadataV1:fieldName=products.taxless_price', - "products_quantity" Array(Int64) COMMENT 'quesmaMetadataV1:fieldName=products.quantity', - "customer_birth_date" Nullable(DateTime64) COMMENT 'quesmaMetadataV1:fieldName=customer_birth_date', - -) -ENGINE = MergeTree -ORDER BY ("@timestamp") - -COMMENT 'created by Quesma'` - - fieldNames := []string{"@timestamp", "attributes_values", "attributes_metadata", "taxful_total_price", "sku", "taxless_total_price", "total_unique_products", "geoip_region_name", "category", "products_created_on", "products_taxful_price", "user", "currency", "day_of_week", "geoip_location_lat", "geoip_city_name", "products__id", "products_discount_amount", "customer_last_name", "total_quantity", "geoip_continent_name", "products_price", "products_sku", "products_discount_percentage", "type", "customer_full_name", "products_min_price", "products_unit_discount_amount", "products_manufacturer", "products_base_unit_price", "day_of_week_i", "manufacturer", "customer_first_name", "products_product_id", "customer_gender", "email", "order_id", "customer_phone", "order_date", "geoip_location_lon", "products_base_price", "products_category", "products_tax_amount", "products_product_name", "customer_id", "event_dataset", "geoip_country_iso_code", "products_taxless_price", "products_quantity", "customer_birth_date"} - table, err := clickhouse.NewTable(q, nil) - assert.NoError(t, err) - assert.Equal(t, len(fieldNames), len(table.Cols)) - for _, fieldName := range fieldNames { - assert.Contains(t, table.Cols, fieldName) - } - assert.Equal(t, "kibana_sample_data_ecommerce", table.Name) - assert.Equal(t, "quesma_cluster", table.ClusterName) -} - -func TestParseLongNestedSchema(t *testing.T) { - q := `CREATE TABLE IF NOT EXISTS "/_monitoring/bulk?system_id=kibana&system_api_version=7&interval=10000ms_2" - ( - "processes" String, - "os" Tuple - ( - "uptime_in_millis" String, - "distro" String, - "cpuacct" Tuple - ( - "control_group" String, - "usage_nanos" String - ), - "distroRelease" String, - "cpu" Tuple - ( - "control_group" String, - "stat" Tuple - ( - "number_of_elapsed_periods" String, - "number_of_times_throttled" String, - "time_throttled_nanos" String - ), - "cfs_quota_micros" String, - "cfs_period_micros" String - ), - "platform" String, - "platformRelease" String, - "load" Tuple - ( - "1m" String, - "5m" String, - "15m" String - ), - "memory" Tuple - ( - "total_in_bytes" String, - "free_in_bytes" String, - "used_in_bytes" String - ) - ), - "concurrent_connections" String, - "requests" Tuple - ( - "disconnects" String, - "total" String - ), - "kibana" Tuple - ( - "name" String, - "index" String, - "host" String, - "transport_address" String, - "version" String, - "snapshot" Bool, - "status" String, - "uuid" String - ), - "elasticsearch_client" Tuple - ( - "totalActiveSockets" String, - "totalIdleSockets" String, - "totalQueuedRequests" String - ), - "response_times" Tuple - ( - "average" String, - "max" String - ), - "process" Tuple - ( - "event_loop_delay" String, - "event_loop_delay_histogram" Tuple - ( - "mean" String, - "exceeds" String, - "stddev" String, - "fromTimestamp" DateTime64, - "lastUpdatedAt" DateTime64, - "percentiles" Tuple - ( - "99" String, - "50" String, - "75" String, - "95" String - ), - "min" String, - "max" String - ), - "event_loop_utilization" Tuple - ( - "active" String, - "idle" String, - "utilization" String - ), - "uptime_in_millis" String, - "memory" Tuple - ( - "heap" Tuple - ( - "total_in_bytes" String, - "used_in_bytes" String, - "size_limit" String - ), - "resident_set_size_in_bytes" String - ) - ), - "timestamp" DateTime64 DEFAULT now64() - ) - ENGINE = MergeTree - ORDER BY (timestamp)` - fieldNames := []string{"processes", "os", "concurrent_connections", "requests", "kibana", "elasticsearch_client", "response_times", "process", "timestamp"} - table, err := clickhouse.NewTable(q, nil) - assert.NoError(t, err) - assert.Equal(t, len(fieldNames), len(table.Cols)) - for _, fieldName := range fieldNames { - assert.Contains(t, table.Cols, fieldName) - } - assert.Equal(t, 3, len(table.Cols["elasticsearch_client"].Type.(clickhouse.MultiValueType).Cols)) - assert.Equal(t, 5, len(table.Cols["process"].Type.(clickhouse.MultiValueType).Cols)) -} - -func Test_parseMultiValueType(t *testing.T) { - tupleQueryPart := []string{"(d DateTime64(3) )", "(d DateTime64(3))"} - for i, tuple := range tupleQueryPart { - t.Run(util.PrettyTestName(tuple, i), func(t *testing.T) { - indexAfterMatch, columns := parseMultiValueType(tuple, 0) - assert.NotEqual(t, -1, indexAfterMatch) - assert.Len(t, columns, 1) - }) - } -} - -func TestParseCreateTableWithNullable(t *testing.T) { - const columnNr = 9 - q := `CREATE TABLE IF NOT EXISTS "logs-generic-default" - ( - "nullable-string" Nullable(String), - "nullable-date-time-1" Nullable(DateTime64(6, 'UTC') ), - "nullable-date-time-2" Nullable(DateTime64), - "nullable-date-time-3" Nullable(DateTime('UTC') ), - "non-nullable-string" String, - "nullable-array" Array(Nullable(String)), - "non-nullable-array" Array(Int64), - "tuple" Tuple(a String, b Nullable(String), c Tuple(c String, d Nullable(UInt128))), - "array-tuple" Array(Tuple(nullable Nullable(String), "non-nullable" String)) - ) - ENGINE = Log` - table, err := clickhouse.NewTable(q, nil) - assert.NoError(t, err) - assert.Equal(t, columnNr, len(table.Cols)) - for _, colName := range []string{"nullable-string", "nullable-date-time-1", "nullable-date-time-2", "nullable-date-time-3"} { - assert.True(t, table.Cols[colName].Type.IsNullable(), colName) - } - for _, colName := range []string{"non-nullable-string", "nullable-array", "non-nullable-array", "tuple", "array-tuple"} { - assert.False(t, table.Cols[colName].Type.IsNullable(), colName) - } - // base types - assert.True(t, table.Cols["nullable-array"].Type.(clickhouse.CompoundType).BaseType.IsNullable()) - assert.False(t, table.Cols["non-nullable-array"].Type.(clickhouse.CompoundType).BaseType.IsNullable()) - - // tuple - assert.False(t, table.Cols["tuple"].Type.(clickhouse.MultiValueType).Cols[0].Type.IsNullable()) - assert.True(t, table.Cols["tuple"].Type.(clickhouse.MultiValueType).Cols[1].Type.IsNullable()) - assert.False(t, table.Cols["tuple"].Type.(clickhouse.MultiValueType).Cols[2].Type.(clickhouse.MultiValueType).Cols[0].Type.IsNullable()) - assert.True(t, table.Cols["tuple"].Type.(clickhouse.MultiValueType).Cols[2].Type.(clickhouse.MultiValueType).Cols[1].Type.IsNullable()) - - // array(tuple) - assert.True(t, table.Cols["array-tuple"].Type.(clickhouse.CompoundType).BaseType.(clickhouse.MultiValueType).Cols[0].Type.IsNullable()) - assert.False(t, table.Cols["array-tuple"].Type.(clickhouse.CompoundType).BaseType.(clickhouse.MultiValueType).Cols[1].Type.IsNullable()) -} - -func IgnoredTestParseCreateTableWithDots(t *testing.T) { - const columnNr = 5 - q := `CREATE TABLE IF NOT EXISTS "my-index-2.3" - ( - "attributes_values" Map(String,String), - "attributes_metadata" Map(String,String), - - "level" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=level', - "@timestamp" DateTime64 DEFAULT now64() COMMENT 'quesmaMetadataV1:fieldName=%40timestamp', - "message" Nullable(String) COMMENT 'quesmaMetadataV1:fieldName=message' - ) - ENGINE = MergeTree - ORDER BY ("@timestamp") - - COMMENT 'created by Quesma'` - table, err := clickhouse.NewTable(q, nil) - assert.NoError(t, err) - assert.Equal(t, columnNr, len(table.Cols)) - assert.Equal(t, "my-index-2.3", table.Name) -} From 642ad75295cea45740b8ce74377afc6468af2b36 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 11:03:24 +0200 Subject: [PATCH 03/12] Replace more tests --- .../elastic_query_dsl/query_parser_test.go | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/platform/parsers/elastic_query_dsl/query_parser_test.go b/platform/parsers/elastic_query_dsl/query_parser_test.go index cae726e3a..14c0b64e1 100644 --- a/platform/parsers/elastic_query_dsl/query_parser_test.go +++ b/platform/parsers/elastic_query_dsl/query_parser_test.go @@ -157,13 +157,14 @@ func TestQueryParserNoFullTextFields(t *testing.T) { // TODO this test gives wrong results?? func TestQueryParserNoAttrsConfig(t *testing.T) { tableName := "logs-generic-default" - table, err := clickhouse.NewTable(`CREATE TABLE `+tableName+` - ( "message" String, "@timestamp" DateTime64(3, 'UTC'), "attributes_values" Map(String,String))) - ENGINE = Memory`, - clickhouse.NewChTableConfigNoAttrs(), - ) - if err != nil { - t.Fatal(err) + table := clickhouse.Table{ + Name: tableName, + Cols: map[string]*clickhouse.Column{ + "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, + "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64(3,'UTC')")}, + "attributes_values": {Name: "attributes_values", Type: clickhouse.NewBaseType("Map(String,String)")}, + }, + Config: clickhouse.NewChTableConfigNoAttrs(), } cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{}} @@ -186,7 +187,7 @@ func TestQueryParserNoAttrsConfig(t *testing.T) { }, }, } - cw := ClickhouseQueryTranslator{Table: table, Ctx: context.Background(), Schema: s.Tables["logs-generic-default"]} + cw := ClickhouseQueryTranslator{Table: &table, Ctx: context.Background(), Schema: s.Tables["logs-generic-default"]} for i, tt := range testdata.TestsSearchNoAttrs { t.Run(util.PrettyTestName(tt.Name, i), func(t *testing.T) { body, parseErr := types.ParseJSON(tt.QueryJson) @@ -270,12 +271,17 @@ func Test_parseSortFields(t *testing.T) { sortFieldNames: []string{"@timestamp", "_doc"}, }, } - table, _ := clickhouse.NewTable(`CREATE TABLE `+tableName+` - ( "@timestamp" DateTime64(3, 'UTC'), "service.name" String, "no_order_field" String, "_table_field_with_underscore" Int64 ) - ENGINE = Memory`, - clickhouse.NewChTableConfigNoAttrs(), - ) - cw := ClickhouseQueryTranslator{Table: table, Ctx: context.Background()} + table := clickhouse.Table{ + Name: tableName, + Cols: map[string]*clickhouse.Column{ + "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64(3,'UTC')")}, + "service.name": {Name: "service.name", Type: clickhouse.NewBaseType("String")}, + "no_order_field": {Name: "no_order_field", Type: clickhouse.NewBaseType("String")}, + "_table_field_with_underscore": {Name: "_table_field_with_underscore", Type: clickhouse.NewBaseType("Int64")}, + }, + Config: clickhouse.NewChTableConfigNoAttrs(), + } + cw := ClickhouseQueryTranslator{Table: &table, Ctx: context.Background()} for i, tt := range tests { t.Run(util.PrettyTestName(tt.name, i), func(t *testing.T) { orderBy, sortFieldNames := cw.parseSortFields(tt.sortMap) From 35fa7758dd8c532694ada96ed0ab29273c55c971 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 11:51:15 +0200 Subject: [PATCH 04/12] More test adapted --- cmd/v2_test_objects.go | 2 +- .../elastic_query_dsl/query_parser_test.go | 24 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cmd/v2_test_objects.go b/cmd/v2_test_objects.go index 2e172c79b..70019e655 100644 --- a/cmd/v2_test_objects.go +++ b/cmd/v2_test_objects.go @@ -393,7 +393,7 @@ func (p *QueryTransformationPipeline) ParseQuery(message any) (*model.ExecutionP DatabaseName: "default", Cols: map[string]*clickhouse.Column{ "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, - "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64(3, 'UTC')")}, + "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64")}, "attributes_values": {Name: "attributes_values", Type: clickhouse.NewBaseType("Map(String,String)")}, }, Config: clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), diff --git a/platform/parsers/elastic_query_dsl/query_parser_test.go b/platform/parsers/elastic_query_dsl/query_parser_test.go index 14c0b64e1..80b6264e4 100644 --- a/platform/parsers/elastic_query_dsl/query_parser_test.go +++ b/platform/parsers/elastic_query_dsl/query_parser_test.go @@ -29,20 +29,22 @@ import ( func TestQueryParserStringAttrConfig(t *testing.T) { logger.InitSimpleLoggerForTestsWarnLevel() tableName := "logs-generic-default" - table, err := clickhouse.NewTable(`CREATE TABLE `+tableName+` - ( "message" String, "@timestamp" DateTime64(3, 'UTC'), "tsAsUInt64" UInt64, "attributes_values" Map(String,String)) - ENGINE = Memory`, - clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), - ) - if err != nil { - t.Fatal(err) + table := clickhouse.Table{ + Name: tableName, + Cols: map[string]*clickhouse.Column{ + "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, + "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64")}, + "tsAsUInt64": {Name: "tsAsUInt64", Type: clickhouse.NewBaseType("UInt64")}, + "attributes_values": {Name: "attributes_values", Type: clickhouse.NewBaseType("Map(String,String)")}, + }, + Config: clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), } cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{}} cfg.IndexConfig["logs-generic-default"] = config.IndexConfiguration{} lm := clickhouse.NewEmptyLogManager(&cfg, nil, diag.NewPhoneHomeEmptyAgent(), clickhouse.NewTableDiscovery(&config.QuesmaConfiguration{}, nil, persistence.NewStaticJSONDatabase())) - lm.AddTableIfDoesntExist(table) + lm.AddTableIfDoesntExist(&table) s := schema.StaticRegistry{ Tables: map[schema.IndexName]schema.Schema{ "logs-generic-default": { @@ -63,7 +65,7 @@ func TestQueryParserStringAttrConfig(t *testing.T) { }, }, } - cw := ClickhouseQueryTranslator{Table: table, Ctx: context.Background(), Schema: s.Tables[schema.IndexName(tableName)]} + cw := ClickhouseQueryTranslator{Table: &table, Ctx: context.Background(), Schema: s.Tables[schema.IndexName(tableName)]} for i, tt := range testdata.TestsSearch { t.Run(util.PrettyTestName(tt.Name, i), func(t *testing.T) { @@ -161,7 +163,7 @@ func TestQueryParserNoAttrsConfig(t *testing.T) { Name: tableName, Cols: map[string]*clickhouse.Column{ "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, - "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64(3,'UTC')")}, + "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64")}, "attributes_values": {Name: "attributes_values", Type: clickhouse.NewBaseType("Map(String,String)")}, }, Config: clickhouse.NewChTableConfigNoAttrs(), @@ -274,7 +276,7 @@ func Test_parseSortFields(t *testing.T) { table := clickhouse.Table{ Name: tableName, Cols: map[string]*clickhouse.Column{ - "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64(3,'UTC')")}, + "@timestamp": {Name: "@timestamp", Type: clickhouse.NewBaseType("DateTime64")}, "service.name": {Name: "service.name", Type: clickhouse.NewBaseType("String")}, "no_order_field": {Name: "no_order_field", Type: clickhouse.NewBaseType("String")}, "_table_field_with_underscore": {Name: "_table_field_with_underscore", Type: clickhouse.NewBaseType("Int64")}, From c335418b548afe340b1278ca198ce797236dc055 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 11:56:54 +0200 Subject: [PATCH 05/12] More test modernization --- .../query_parser_range_test.go | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/platform/parsers/elastic_query_dsl/query_parser_range_test.go b/platform/parsers/elastic_query_dsl/query_parser_range_test.go index 1c9b74a8f..e1b814000 100644 --- a/platform/parsers/elastic_query_dsl/query_parser_range_test.go +++ b/platform/parsers/elastic_query_dsl/query_parser_range_test.go @@ -14,7 +14,7 @@ import ( type parseRangeTest struct { name string rangePartOfQuery QueryMap - createTableQuery string + table clickhouse.Table expectedWhere string } @@ -28,9 +28,14 @@ var parseRangeTests = []parseRangeTest{ "lte": "2024-02-09T13:47:16.029Z", }, }, - `CREATE TABLE ` + tableName + ` - ( "message" String, "timestamp" DateTime64(3, 'UTC') ) - ENGINE = Memory`, + clickhouse.Table{ + Name: tableName, + Cols: map[string]*clickhouse.Column{ + "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, + "timestamp": {Name: "timestamp", Type: clickhouse.NewBaseType("DateTime64")}, + }, + Config: clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), + }, `("timestamp">=fromUnixTimestamp64Milli(1706881636029) AND "timestamp"<=fromUnixTimestamp64Milli(1707486436029))`, }, { @@ -42,9 +47,14 @@ var parseRangeTests = []parseRangeTest{ "lte": "2024-02-09T13:47:16.029Z", }, }, - `CREATE TABLE ` + tableName + ` - ( "message" String, "timestamp" DateTime ) - ENGINE = Memory`, + clickhouse.Table{ + Name: tableName, + Cols: map[string]*clickhouse.Column{ + "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, + "timestamp": {Name: "timestamp", Type: clickhouse.NewBaseType("DateTime")}, + }, + Config: clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), + }, `("timestamp">=fromUnixTimestamp(1706881636) AND "timestamp"<=fromUnixTimestamp(1707486436))`, }, { @@ -54,9 +64,14 @@ var parseRangeTests = []parseRangeTest{ "gt": "100", }, }, - `CREATE TABLE ` + tableName + ` - ( "message" String, "timestamp" DateTime, "time_taken" UInt32 ) - ENGINE = Memory`, + clickhouse.Table{ + Name: tableName, + Cols: map[string]*clickhouse.Column{ + "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, + "time_taken": {Name: "time_taken", Type: clickhouse.NewBaseType("UInt32")}, + }, + Config: clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), + }, `"time_taken">100`, }, { @@ -68,9 +83,14 @@ var parseRangeTests = []parseRangeTest{ "lte": "2024-02-09T13:47:16", }, }, - `CREATE TABLE ` + tableName + ` - ( "message" String, "timestamp" DateTime64(3, 'UTC') ) - ENGINE = Memory`, + clickhouse.Table{ + Name: tableName, + Cols: map[string]*clickhouse.Column{ + "message": {Name: "message", Type: clickhouse.NewBaseType("String")}, + "timestamp": {Name: "timestamp", Type: clickhouse.NewBaseType("DateTime64")}, + }, + Config: clickhouse.NewNoTimestampOnlyStringAttrCHConfig(), + }, `("timestamp">=fromUnixTimestamp64Milli(1706881636000) AND "timestamp"<=fromUnixTimestamp64Milli(1707486436000))`, }, } @@ -96,12 +116,7 @@ func Test_parseRange(t *testing.T) { } for i, test := range parseRangeTests { t.Run(util.PrettyTestName(test.name, i), func(t *testing.T) { - table, err := clickhouse.NewTable(test.createTableQuery, clickhouse.NewNoTimestampOnlyStringAttrCHConfig()) - if err != nil { - t.Fatal(err) - } - assert.NoError(t, err) - cw := ClickhouseQueryTranslator{Table: table, Ctx: context.Background(), Schema: s.Tables[schema.IndexName(tableName)]} + cw := ClickhouseQueryTranslator{Table: &test.table, Ctx: context.Background(), Schema: s.Tables[schema.IndexName(tableName)]} simpleQuery := cw.parseRange(test.rangePartOfQuery) assert.Equal(t, test.expectedWhere, simpleQuery.WhereClauseAsString()) From 29a7f251d5dc83b518042b7e132e2b2abb98c308 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 12:03:03 +0200 Subject: [PATCH 06/12] More test modernization --- platform/ingest/insert_test.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/platform/ingest/insert_test.go b/platform/ingest/insert_test.go index 874c38485..59a497534 100644 --- a/platform/ingest/insert_test.go +++ b/platform/ingest/insert_test.go @@ -175,9 +175,29 @@ func TestAutomaticTableCreationAtInsert(t *testing.T) { columns := columnsWithIndexes(columnsToString(columnsFromJson, columnsFromSchema, encodings, tableName), Indexes(types.MustJSON(tt.insertJson))) query := createTableQuery(tableName, columns, tableConfig) - table, err := clickhouse.NewTable(query, tableConfig) - assert.NoError(t, err) - query = addOurFieldsToCreateTableQuery(query, tableConfig, table) + tableColumns := make(map[string]*clickhouse.Column) + for _, c := range columnsFromJson { + tableColumns[c.ClickHouseColumnName] = &clickhouse.Column{ + Name: c.ClickHouseColumnName, + Type: clickhouse.NewBaseType(c.ClickHouseType), + } + } + for _, c := range columnsFromSchema { + if _, exists := tableColumns[c.ClickHouseColumnName]; !exists { + tableColumns[c.ClickHouseColumnName] = &clickhouse.Column{ + Name: c.ClickHouseColumnName, + Type: clickhouse.NewBaseType(c.ClickHouseType), + } + } + } + + table := clickhouse.Table{ + Name: tableName, + Cols: tableColumns, + Config: tableConfig, + } + + query = addOurFieldsToCreateTableQuery(query, tableConfig, &table) // check if CREATE TABLE string is OK queryByLine := strings.Split(query, "\n") @@ -200,7 +220,7 @@ func TestAutomaticTableCreationAtInsert(t *testing.T) { if tableInMemory != nil { needCreate = false } - noSuchTable := ip.ip.AddTableIfDoesntExist(table) + noSuchTable := ip.ip.AddTableIfDoesntExist(&table) assert.Equal(t, needCreate, noSuchTable) // and Created is set to true From 42a5e6174fd97ad6296ecf540f3d667eb0338790 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 12:17:21 +0200 Subject: [PATCH 07/12] WIP --- platform/ingest/insert_test.go | 26 +++----------------- platform/ingest/processor.go | 43 +++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/platform/ingest/insert_test.go b/platform/ingest/insert_test.go index 59a497534..7f31a929c 100644 --- a/platform/ingest/insert_test.go +++ b/platform/ingest/insert_test.go @@ -175,29 +175,9 @@ func TestAutomaticTableCreationAtInsert(t *testing.T) { columns := columnsWithIndexes(columnsToString(columnsFromJson, columnsFromSchema, encodings, tableName), Indexes(types.MustJSON(tt.insertJson))) query := createTableQuery(tableName, columns, tableConfig) - tableColumns := make(map[string]*clickhouse.Column) - for _, c := range columnsFromJson { - tableColumns[c.ClickHouseColumnName] = &clickhouse.Column{ - Name: c.ClickHouseColumnName, - Type: clickhouse.NewBaseType(c.ClickHouseType), - } - } - for _, c := range columnsFromSchema { - if _, exists := tableColumns[c.ClickHouseColumnName]; !exists { - tableColumns[c.ClickHouseColumnName] = &clickhouse.Column{ - Name: c.ClickHouseColumnName, - Type: clickhouse.NewBaseType(c.ClickHouseType), - } - } - } - - table := clickhouse.Table{ - Name: tableName, - Cols: tableColumns, - Config: tableConfig, - } + table := ip.ip.createTableObject(tableName, columnsFromJson, columnsFromSchema, tableConfig) - query = addOurFieldsToCreateTableQuery(query, tableConfig, &table) + query = addOurFieldsToCreateTableQuery(query, tableConfig, table) // check if CREATE TABLE string is OK queryByLine := strings.Split(query, "\n") @@ -220,7 +200,7 @@ func TestAutomaticTableCreationAtInsert(t *testing.T) { if tableInMemory != nil { needCreate = false } - noSuchTable := ip.ip.AddTableIfDoesntExist(&table) + noSuchTable := ip.ip.AddTableIfDoesntExist(table) assert.Equal(t, needCreate, noSuchTable) // and Created is set to true diff --git a/platform/ingest/processor.go b/platform/ingest/processor.go index ec29990c7..1d5dd9280 100644 --- a/platform/ingest/processor.go +++ b/platform/ingest/processor.go @@ -178,17 +178,39 @@ func (ip *IngestProcessor) Count(ctx context.Context, table string) (int64, erro return count, nil } -func (ip *IngestProcessor) createTableObjectAndAttributes(ctx context.Context, query string, config *chLib.ChTableConfig, name string, tableDefinitionChangeOnly bool) (string, error) { - table, err := chLib.NewTable(query, config) - if err != nil { - return "", err +func (ip *IngestProcessor) createTableObject(tableName string, columnsFromJson []CreateTableEntry, columnsFromSchema map[schema.FieldName]CreateTableEntry, tableConfig *chLib.ChTableConfig) *chLib.Table { + tableColumns := make(map[string]*chLib.Column) + for _, c := range columnsFromJson { + tableColumns[c.ClickHouseColumnName] = &chLib.Column{ + Name: c.ClickHouseColumnName, + Type: chLib.NewBaseType(c.ClickHouseType), + } + } + for _, c := range columnsFromSchema { + if _, exists := tableColumns[c.ClickHouseColumnName]; !exists { + tableColumns[c.ClickHouseColumnName] = &chLib.Column{ + Name: c.ClickHouseColumnName, + Type: chLib.NewBaseType(c.ClickHouseType), + } + } } + table := chLib.Table{ + Name: tableName, + Cols: tableColumns, + Config: tableConfig, + } + + return &table +} + +func (ip *IngestProcessor) createTableObjectAndAttributes(ctx context.Context, tableName string, columnsFromJson []CreateTableEntry, columnsFromSchema map[schema.FieldName]CreateTableEntry, tableConfig *chLib.ChTableConfig, tableDefinitionChangeOnly bool) (*chLib.Table, error) { + table := ip.createTableObject(tableName, columnsFromJson, columnsFromSchema, tableConfig) + // This is a HACK. // CreateQueryParser assumes that the table name is in the form of "database.table" // in this case we don't have a database name, so we need to add it if tableDefinitionChangeOnly { - table.Name = name table.DatabaseName = "" table.Comment = "Definition only. This is not a real table." table.VirtualTable = true @@ -197,10 +219,11 @@ func (ip *IngestProcessor) createTableObjectAndAttributes(ctx context.Context, q // if exists only then createTable noSuchTable := ip.AddTableIfDoesntExist(table) if !noSuchTable { - return "", fmt.Errorf("table %s already exists", table.Name) + return nil, fmt.Errorf("table %s already exists", table.Name) } - return addOurFieldsToCreateTableQuery(query, config, table), nil + // addOurFieldsToCreateTableQuery(query, config, table) + return table, nil } func findSchemaPointer(schemaRegistry schema.Registry, tableName string) *schema.Schema { @@ -663,14 +686,16 @@ func (ip *IngestProcessor) processInsertQuery(ctx context.Context, columnsAsString := columnsWithIndexes(columnsToString(columnsFromJson, columnsFromSchema, ip.schemaRegistry.GetFieldEncodings(), tableName), Indexes(transformedJsons[0])) createTableCmd = createTableQuery(tableName, columnsAsString, tableConfig) var err error - createTableCmd, err = ip.createTableObjectAndAttributes(ctx, createTableCmd, tableConfig, tableName, tableDefinitionChangeOnly) + table, err = ip.createTableObjectAndAttributes(ctx, tableName, columnsFromJson, columnsFromSchema, tableConfig, tableDefinitionChangeOnly) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error createTableObjectAndAttributes, can't create table: %v", err) return nil, err } else { + // Likely we want to remove below line + createTableCmd = addOurFieldsToCreateTableQuery(createTableCmd, tableConfig, table) logger.InfoWithCtx(ctx).Msgf("created table '%s' with query: %s", tableName, createTableCmd) } - // Set pointer to table after creating it + // Set pointer to table after creating it, TODO: probably to remove table = ip.FindTable(tableName) } if table == nil { From 684efbda84b4b9cf3719507cffedad1b16e83715 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 13:13:22 +0200 Subject: [PATCH 08/12] Make it working --- platform/clickhouse/table_discovery.go | 8 ++++---- platform/clickhouse/table_discovery_test.go | 4 ++-- platform/ingest/parser.go | 4 ++++ platform/ingest/processor.go | 18 +++++++++++++++--- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/platform/clickhouse/table_discovery.go b/platform/clickhouse/table_discovery.go index 502a938b5..20164e9b2 100644 --- a/platform/clickhouse/table_discovery.go +++ b/platform/clickhouse/table_discovery.go @@ -393,7 +393,7 @@ func (td *tableDiscovery) populateTableDefinitions(configuredTables map[string]d } } - column := resolveColumn(col, columnMeta.colType) + column := ResolveColumn(col, columnMeta.colType) if column != nil { column.Comment = columnMeta.comment column.Origin = columnMeta.origin @@ -477,7 +477,7 @@ func (td *tableDiscovery) TableDefinitions() *TableMap { return td.tableDefinitions.Load() } -func resolveColumn(colName, colType string) *Column { +func ResolveColumn(colName, colType string) *Column { isNullable := false if isNullableType(colType) { isNullable = true @@ -491,7 +491,7 @@ func resolveColumn(colName, colType string) *Column { arrayType = strings.TrimSuffix(strings.TrimPrefix(arrayType, "Nullable("), ")") } if isArrayType(arrayType) { - innerColumn := resolveColumn("inner", arrayType) + innerColumn := ResolveColumn("inner", arrayType) if innerColumn == nil { logger.Warn().Msgf("invalid inner array type for column %s, %s", colName, colType) return nil @@ -514,7 +514,7 @@ func resolveColumn(colName, colType string) *Column { }, } } else if isTupleType(arrayType) { - tupleColumn := resolveColumn("Tuple", arrayType) + tupleColumn := ResolveColumn("Tuple", arrayType) if tupleColumn == nil { logger.Warn().Msgf("invalid tuple type for column %s, %s", colName, colType) return nil diff --git a/platform/clickhouse/table_discovery_test.go b/platform/clickhouse/table_discovery_test.go index 0282d0549..c28d5d2d1 100644 --- a/platform/clickhouse/table_discovery_test.go +++ b/platform/clickhouse/table_discovery_test.go @@ -146,7 +146,7 @@ func Test_resolveColumn(t *testing.T) { for i, tt := range tests { t.Run(util.PrettyTestName(tt.name, i), func(t *testing.T) { - assert.Equalf(t, tt.want, resolveColumn(tt.args.colName, tt.args.colType), "resolveColumn(%v, %v)", tt.args.colName, tt.args.colType) + assert.Equalf(t, tt.want, ResolveColumn(tt.args.colName, tt.args.colType), "ResolveColumn(%v, %v)", tt.args.colName, tt.args.colType) }) } } @@ -300,7 +300,7 @@ func Test_resolveColumn_Nullable(t *testing.T) { for i, tt := range tests { t.Run(util.PrettyTestName(tt.name, i), func(t *testing.T) { - assert.Equalf(t, tt.want, resolveColumn(tt.args.colName, tt.args.colType), "resolveColumn(%v, %v)", tt.args.colName, tt.args.colType) + assert.Equalf(t, tt.want, ResolveColumn(tt.args.colName, tt.args.colType), "ResolveColumn(%v, %v)", tt.args.colName, tt.args.colType) }) } } diff --git a/platform/ingest/parser.go b/platform/ingest/parser.go index 3dedbbb10..f9f5ccf31 100644 --- a/platform/ingest/parser.go +++ b/platform/ingest/parser.go @@ -239,8 +239,12 @@ func DifferenceMap(sm SchemaMap, t *clickhouse.Table) SchemaMap { } } + fmt.Println("JM 3 table.Cols", t.Cols, sm) + for name, v := range sm { col, ok := t.Cols[name] + fmt.Println("JM 3.1 name, ok", name, ok) + if !ok { mDiff[name] = v } else if mNested, ok := v.(SchemaMap); ok { diff --git a/platform/ingest/processor.go b/platform/ingest/processor.go index 1d5dd9280..34340b59a 100644 --- a/platform/ingest/processor.go +++ b/platform/ingest/processor.go @@ -179,18 +179,27 @@ func (ip *IngestProcessor) Count(ctx context.Context, table string) (int64, erro } func (ip *IngestProcessor) createTableObject(tableName string, columnsFromJson []CreateTableEntry, columnsFromSchema map[schema.FieldName]CreateTableEntry, tableConfig *chLib.ChTableConfig) *chLib.Table { + resolveType := func(name, colType string) chLib.Type { + if strings.Contains(colType, " DEFAULT") { + // Remove DEFAULT clause from the type + colType = strings.Split(colType, " DEFAULT")[0] + } + resCol := chLib.ResolveColumn(name, colType) + return resCol.Type + } + tableColumns := make(map[string]*chLib.Column) for _, c := range columnsFromJson { tableColumns[c.ClickHouseColumnName] = &chLib.Column{ Name: c.ClickHouseColumnName, - Type: chLib.NewBaseType(c.ClickHouseType), + Type: resolveType(c.ClickHouseColumnName, c.ClickHouseType), } } for _, c := range columnsFromSchema { if _, exists := tableColumns[c.ClickHouseColumnName]; !exists { tableColumns[c.ClickHouseColumnName] = &chLib.Column{ Name: c.ClickHouseColumnName, - Type: chLib.NewBaseType(c.ClickHouseType), + Type: resolveType(c.ClickHouseColumnName, c.ClickHouseType), } } } @@ -222,7 +231,6 @@ func (ip *IngestProcessor) createTableObjectAndAttributes(ctx context.Context, t return nil, fmt.Errorf("table %s already exists", table.Name) } - // addOurFieldsToCreateTableQuery(query, config, table) return table, nil } @@ -697,6 +705,9 @@ func (ip *IngestProcessor) processInsertQuery(ctx context.Context, } // Set pointer to table after creating it, TODO: probably to remove table = ip.FindTable(tableName) + tableAlt, _ := chLib.NewTable(createTableCmd, tableConfig) + fmt.Println("JM 1 table", table, "tableOrig", tableAlt) + } if table == nil { return nil, fmt.Errorf("table %s not found", tableName) @@ -711,6 +722,7 @@ func (ip *IngestProcessor) processInsertQuery(ctx context.Context, return nil, fmt.Errorf("error preprocessJsons: %v", err) } for i, preprocessedJson := range validatedJsons { + fmt.Println("JM 2 table.Cols", table.Cols) alter, onlySchemaFields, nonSchemaFields, err := ip.GenerateIngestContent(table, preprocessedJson, invalidJsons[i], tableConfig, encodings) From 981c743542fd0ba23d61a2dc932d4cf915874508 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 13:14:49 +0200 Subject: [PATCH 09/12] Remove hacks --- platform/ingest/parser.go | 4 ---- platform/ingest/processor.go | 8 +------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/platform/ingest/parser.go b/platform/ingest/parser.go index f9f5ccf31..3dedbbb10 100644 --- a/platform/ingest/parser.go +++ b/platform/ingest/parser.go @@ -239,12 +239,8 @@ func DifferenceMap(sm SchemaMap, t *clickhouse.Table) SchemaMap { } } - fmt.Println("JM 3 table.Cols", t.Cols, sm) - for name, v := range sm { col, ok := t.Cols[name] - fmt.Println("JM 3.1 name, ok", name, ok) - if !ok { mDiff[name] = v } else if mNested, ok := v.(SchemaMap); ok { diff --git a/platform/ingest/processor.go b/platform/ingest/processor.go index 34340b59a..05b00cbf7 100644 --- a/platform/ingest/processor.go +++ b/platform/ingest/processor.go @@ -187,7 +187,7 @@ func (ip *IngestProcessor) createTableObject(tableName string, columnsFromJson [ resCol := chLib.ResolveColumn(name, colType) return resCol.Type } - + tableColumns := make(map[string]*chLib.Column) for _, c := range columnsFromJson { tableColumns[c.ClickHouseColumnName] = &chLib.Column{ @@ -703,11 +703,6 @@ func (ip *IngestProcessor) processInsertQuery(ctx context.Context, createTableCmd = addOurFieldsToCreateTableQuery(createTableCmd, tableConfig, table) logger.InfoWithCtx(ctx).Msgf("created table '%s' with query: %s", tableName, createTableCmd) } - // Set pointer to table after creating it, TODO: probably to remove - table = ip.FindTable(tableName) - tableAlt, _ := chLib.NewTable(createTableCmd, tableConfig) - fmt.Println("JM 1 table", table, "tableOrig", tableAlt) - } if table == nil { return nil, fmt.Errorf("table %s not found", tableName) @@ -722,7 +717,6 @@ func (ip *IngestProcessor) processInsertQuery(ctx context.Context, return nil, fmt.Errorf("error preprocessJsons: %v", err) } for i, preprocessedJson := range validatedJsons { - fmt.Println("JM 2 table.Cols", table.Cols) alter, onlySchemaFields, nonSchemaFields, err := ip.GenerateIngestContent(table, preprocessedJson, invalidJsons[i], tableConfig, encodings) From f3360a159f5ea4d9385395283291879061d64ef3 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 13:19:14 +0200 Subject: [PATCH 10/12] Remove most of parse create table --- platform/clickhouse/parserCreateTable.go | 244 ----------------------- platform/clickhouse/schema.go | 10 - 2 files changed, 254 deletions(-) diff --git a/platform/clickhouse/parserCreateTable.go b/platform/clickhouse/parserCreateTable.go index 9c6c6065e..3afb81427 100644 --- a/platform/clickhouse/parserCreateTable.go +++ b/platform/clickhouse/parserCreateTable.go @@ -4,7 +4,6 @@ package clickhouse import ( "github.com/QuesmaOrg/quesma/platform/logger" - "strings" "unicode" ) @@ -43,16 +42,6 @@ func parseMaybeAndForget(q string, i int, s string) (int, bool) { return i, false } -func parseMaybeAndForgetMultiple(q string, i int, ss []string) (int, bool) { - for _, s := range ss { - i2, ok := parseMaybeAndForget(q, i, s) - if ok { - return i2, true - } - } - return i, false -} - func isGoodIdentChar(r rune) bool { return !unicode.IsSpace(r) && r != ')' && r != '"' && r != '`' && r != ',' && r != '(' } @@ -127,82 +116,6 @@ func parseIdentWithBrackets(q string, i int) (int, string) { return -1, "" } -func parseColumn(q string, i int) (int, Column) { - col := Column{} - i = omitWhitespace(q, i) - // name - quote := `"` - i2 := parseExact(q, i, quote) - if i2 == -1 { - quote = "`" - i2 = parseExact(q, i, quote) - if i2 == -1 { - return -1, col - } - } - i, col.Name = parseIdent(q, i2) - if i == -1 { - return -1, col - } - i = parseExact(q, i, quote) - // type - if i == -1 { - return -1, col - } - i, col.Type = parseNullable(q, i) - if i == -1 { - return -1, col - } - - // NULL | NOT NULL - i = omitWhitespace(q, i) - i, _ = parseMaybeAndForgetMultiple(q, i, []string{"NULL", "NOT NULL"}) - - // DEFAULT | MATERIALIZED | EPHEMERAL | ALIAS expr - i = omitWhitespace(q, i) - i, ok := parseMaybeAndForgetMultiple(q, i, []string{"DEFAULT", "MATERIALIZED", "EPHEMERAL", "ALIAS"}) - if ok { - i = omitWhitespace(q, i) - i = parseExpr(q, i) - if i == -1 { - return -1, col - } - i = omitWhitespace(q, i) - } - - // CODEC - if i+5 < len(q) && q[i:i+5] == "CODEC" { - i, col.Codec = parseCodec(q, i) - i = omitWhitespace(q, i) - } - - // TTL - if i+3 < len(q) && q[i:i+3] == "TTL" { - i = omitWhitespace(q, i+3) - i = parseExpr(q, i) - if i == -1 { - return -1, col - } - i = omitWhitespace(q, i) - } - - // COMMENT - if i+7 < len(q) && q[i:i+7] == "COMMENT" { - // TODO should be good enough for now - for { - i++ - if q[i] == ',' { - break - } - } - } - - if i == -1 || i >= len(q) || (q[i] != ',' && q[i] != ')') { - return -1, col - } - return i, col -} - func parseType(q string, i int) (int, Type) { i2, name := parseIdent(q, i) if i == -1 { @@ -288,160 +201,3 @@ func parseMultiValueType(q string, i int) (int, []*Column) { i = omitWhitespace(q, j+1) } } - -func parseCodec(q string, i int) (int, Codec) { - b := i - i = parseExact(q, i, "CODEC") - if i == -1 { - return -1, Codec{} - } - i = omitWhitespace(q, i) - i = parseExact(q, i, "(") - bracketsCnt := 1 - for i < len(q) && bracketsCnt > 0 { - if q[i] == '(' { - bracketsCnt++ - } else if q[i] == ')' { - bracketsCnt-- - } - i++ - } - if i >= len(q) { - return -1, Codec{} - } - return i, Codec{Name: q[b:i]} -} - -// Kind of hackish, but should work 100% of the time, unless CODEC/TTL/COMMENT -// can be used in expressions (I'd assume they can't) -func parseExpr(q string, i int) int { - bracketsCnt := 0 - for i < len(q) { - if q[i] == '(' { - bracketsCnt++ - } else if q[i] == ')' { - bracketsCnt-- - } - if bracketsCnt < 0 { - return i - } - if bracketsCnt == 0 { - if q[i] == ',' { - return i - } - _, ok := parseMaybeAndForgetMultiple(q, i, []string{"CODEC", "TTL", "COMMENT"}) - if ok { - return i - } - if q[i] == ')' { - i2 := omitWhitespace(q, i+1) - if parseExact(q, i2, "ENGINE") != -1 { - return i - } - } - } - i = omitWhitespace(q, i+1) - } - return -1 -} - -// 0 = success, -// > 0 - fail, char index where failed -// Tuples can be unnamed. In this case they are not supported yet, as I'm not sure -// if it's worth adding right now. -func ParseCreateTable(q string) (*Table, int) { - t := Table{} - - // parse header - i := parseExact(q, 0, "CREATE TABLE ") - if i == -1 { - return &t, 1 - } - i, _ = parseMaybeAndForget(q, i, "IF NOT EXISTS ") - - // parse [db.]table_name - i = omitWhitespace(q, i) - i2 := parseExact(q, i, `"`) - quote := i2 != -1 - if quote { - i = i2 - } - i2, ident := parseIdent(q, i) // ident = db name or table name - if i2 == -1 { - return &t, i - } - if strings.Contains(ident, ".") { // If it has ".", it means it is DB name - split := strings.Split(ident, ".") - if len(split) > 1 { - t.Name = strings.Join(split[1:], ".") - } - t.DatabaseName = split[0] - } else { - t.Name = ident - } - if quote { - i2 = parseExact(q, i2, `"`) - if i2 == -1 { - return &t, i - } - } - - // parse [ON CLUSTER cluster_name] - i3 := parseExact(q, i2, "ON CLUSTER ") - if i3 != -1 { - i3 = omitWhitespace(q, i3) - i4, _ := parseMaybeAndForget(q, i3, `"`) // cluster name can be quoted, but doesn't have to - if i4 != -1 { - i3 = i4 - } - i4, ident := parseIdent(q, i3) - if i4 == -1 { - return &t, i3 - } - t.ClusterName = ident - if i4 != -1 { - i4, _ = parseMaybeAndForget(q, i4, `"`) - if i4 == -1 { - return &t, i3 - } - } - i2 = i4 - } - - i3 = parseExact(q, i2, "(") - if i3 == -1 { - return &t, i2 - } - - // parse columns - t.Cols = make(map[string]*Column) - for { - i = omitWhitespace(q, i3) - if parseExact(q, i, "INDEX") != -1 { - return &t, 0 - } - i, col := parseColumn(q, i3) - if i == -1 { - return &t, i3 - } - t.Cols[col.Name] = &col - i2 = omitWhitespace(q, i) - if i2 == -1 { - return &t, i - } - if q[i2] == ')' { - return &t, 0 - } else if q[i2] != ',' { - return &t, i2 - } else { - i3 = omitWhitespace(q, i2+1) - if i3 == -1 { - return &t, i2 + 1 - } else if q[i3] == ')' { - return &t, 0 - } else { - i3 = i2 + 1 - } - } - } -} diff --git a/platform/clickhouse/schema.go b/platform/clickhouse/schema.go index de46f15d1..8a47511bf 100644 --- a/platform/clickhouse/schema.go +++ b/platform/clickhouse/schema.go @@ -311,16 +311,6 @@ func NewType(value any, valueOrigin string) (Type, error) { return nil, fmt.Errorf("unsupported type '%T' of value: %v (origin: %s)", value, value, valueOrigin) } -func NewTable(createTableQuery string, config *ChTableConfig) (*Table, error) { - t, i := ParseCreateTable(createTableQuery) - t.Config = config - if i == 0 { - return t, nil - } else { - return t, fmt.Errorf("error parsing query at character %d, query: %s", i, createTableQuery) - } -} - // NewEmptyTable is used only in tests func NewEmptyTable(tableName string) *Table { return &Table{Name: tableName, Config: NewChTableConfigNoAttrs()} From dad983b7826bb0b81089649b87058248676c908f Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 14:05:05 +0200 Subject: [PATCH 11/12] Remove one more file --- platform/ingest/parserCreateTable.go | 204 --------------------------- 1 file changed, 204 deletions(-) delete mode 100644 platform/ingest/parserCreateTable.go diff --git a/platform/ingest/parserCreateTable.go b/platform/ingest/parserCreateTable.go deleted file mode 100644 index 0b46794dc..000000000 --- a/platform/ingest/parserCreateTable.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright Quesma, licensed under the Elastic License 2.0. -// SPDX-License-Identifier: Elastic-2.0 -package ingest - -import ( - "github.com/QuesmaOrg/quesma/platform/clickhouse" - "github.com/QuesmaOrg/quesma/platform/logger" - "unicode" -) - -func omitWhitespace(q string, i int) int { - for i < len(q) && unicode.IsSpace(rune(q[i])) { - i++ - } - if i >= len(q) { - return -1 - } - return i -} - -// Omits whitespaces, then len(s) characters in 'q' need to match 's' -// Returns -1 if not matched, otherwise returns index after the match -func parseExact(q string, i int, s string) int { - i = omitWhitespace(q, i) - if i+len(s) > len(q) { - return -1 - } - if q[i:i+len(s)] == s { - return i + len(s) - } - return -1 -} - -// bool -> if found -func parseMaybeAndForget(q string, i int, s string) (int, bool) { - i = omitWhitespace(q, i) - if i+len(s) > len(q) { - return i, false - } - if q[i:i+len(s)] == s { - return i + len(s), true - } - return i, false -} - -func isGoodIdentChar(r rune) bool { - return !unicode.IsSpace(r) && r != ')' && r != '"' && r != '`' && r != ',' && r != '(' -} - -// TODO idents starting with digit accepted, shouldn't probably be -// parse identificator in q[i:] -func parseIdent(q string, i int) (int, string) { - i = omitWhitespace(q, i) - if i >= len(q) { - return -1, "" - } - if !isGoodIdentChar(rune(q[i])) { - return -1, "" - } - j := i + 1 - for j < len(q) && isGoodIdentChar(rune(q[j])) { - j++ - } - return j, q[i:j] -} - -func parseNullable(q string, i int) (int, clickhouse.Type) { - i = omitWhitespace(q, i) - if q[i] == 'N' { - i, ok := parseMaybeAndForget(q, i, "Nullable") - if ok { - i = parseExact(q, i, "(") - if i == -1 { - return -1, nil - } - i, ident := parseType(q, i) - if i == -1 { - return -1, nil - } - i = parseExact(q, i, ")") - if i == -1 { - return -1, nil - } - typeAsBaseType, ok := ident.(clickhouse.BaseType) - if ok { - typeAsBaseType.Nullable = true - return i, typeAsBaseType - } else { - logger.Warn().Msgf("Only BaseTypes can be Nullable! Here type is not BaseType, but %T", ident) - } - return i, ident - } - } - return parseType(q, i) -} - -// Returns -1 if not matched, otherwise returns (index after the match, ident) -func parseIdentWithBrackets(q string, i int) (int, string) { - i = omitWhitespace(q, i) - if i >= len(q) { - return -1, "" - } - b, e := i, i - bracketsCnt := 0 - for i < len(q) { - if q[i] == '(' { - e = i - bracketsCnt++ - } else if q[i] == ')' { - bracketsCnt-- - } - if bracketsCnt == 0 && (q[i] == ' ' || q[i] == ',' || q[i] == ')') { - return i + 1, q[b:e] - } - i++ - } - return -1, "" -} - -func parseType(q string, i int) (int, clickhouse.Type) { - i2, name := parseIdent(q, i) - if i == -1 { - return -1, nil - } - switch name { - case "Array": - i, baseType := parseCompoundType(q, i2) - if i == -1 { - return -1, nil - } - return i, clickhouse.CompoundType{Name: name, BaseType: baseType} - case "Tuple", "Nested": - i, types := parseMultiValueType(q, i2) - if i == -1 { - return -1, nil - } - return i, clickhouse.MultiValueType{Name: name, Cols: types} - } - if parseExact(q, i2, "(") != -1 { - i, name = parseIdentWithBrackets(q, i) - if i == -1 { - return -1, nil - } - return i, clickhouse.NewBaseType(name) - } else { - return i2, clickhouse.NewBaseType(name) - } -} - -func parseCompoundType(q string, i int) (int, clickhouse.Type) { - i = parseExact(q, i, "(") - if i == -1 { - return -1, nil - } - i, typ := parseNullable(q, i) - if i == -1 { - return -1, nil - } - i = parseExact(q, i, ")") - if i == -1 { - return -1, nil - } - return i, typ -} - -// parseMultiValueType returns -1 if failed, otherwise (index after the match, []*Column) -// TO THINK: subcolumns shouldn't have codecs? Maybe fix it somehow -// TODO maybe merge with 'parseColumn'? Can wait, for now it works as it is. -func parseMultiValueType(q string, i int) (int, []*clickhouse.Column) { - i = parseExact(q, i, "(") - if i == -1 { - return -1, nil - } - var subColumns []*clickhouse.Column - for { - i = omitWhitespace(q, i) - quote := " " - if q[i] == '"' || q[i] == '`' { - quote = string(q[i]) - i++ - } - j, name := parseIdent(q, i) - if j == -1 || (quote != " " && string(q[j]) != quote) { - return -1, nil - } - if quote != " " { - j++ - } - j = omitWhitespace(q, j) - j, typ := parseNullable(q, j) - if j == -1 { - return -1, nil - } - subColumns = append(subColumns, &clickhouse.Column{Name: name, Type: typ}) - j = omitWhitespace(q, j) - if q[j] == ')' { - return j + 1, subColumns - } - if q[j] != ',' { - return -1, nil - } - i = omitWhitespace(q, j+1) - } -} From 081c46bceef8ffa70296552e2e270a3452472786 Mon Sep 17 00:00:00 2001 From: Jacek Migdal Date: Thu, 5 Jun 2025 14:51:26 +0200 Subject: [PATCH 12/12] Add bash script --- bin/it.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/it.sh b/bin/it.sh index 11a255646..a26300b4b 100755 --- a/bin/it.sh +++ b/bin/it.sh @@ -7,4 +7,10 @@ bin/build-image.sh cd ci/it -go test -v +if [ -n "$1" ]; then + # Run only tests matching the pattern + go test -v -run "$1" +else + # Run all tests + go test -v +fi