From f9306f7ef564ce469d035c3e20446ebf8d07c475 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 27 Jun 2025 15:38:08 +0200 Subject: [PATCH 01/54] Fill table name --- platform/ingest/hydrolixlowerer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 3ef14c0cd..e9d36ce5c 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -3,6 +3,7 @@ package ingest import ( + "fmt" chLib "github.com/QuesmaOrg/quesma/platform/clickhouse" "github.com/QuesmaOrg/quesma/platform/persistence" "github.com/QuesmaOrg/quesma/platform/schema" @@ -29,10 +30,10 @@ func (l *HydrolixLowerer) LowerToDDL(validatedJsons []types.JSON, _ = preprocessedJson } - result := []string{`{ + result := []string{fmt.Sprintf(`{ "schema": { "project": "", - "name": "test_index", + "name": %s, "time_column": "ingest_time", "columns": [ { "name": "new_field", "type": "string" }, @@ -49,7 +50,7 @@ func (l *HydrolixLowerer) LowerToDDL(validatedJsons []types.JSON, "new_field": "bar" } ] -}`} +}`, table.Name)} return result, nil } From c55d8e4117d239b03f4b29be5eade990f5e8b700 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 27 Jun 2025 16:11:32 +0200 Subject: [PATCH 02/54] Populate more content --- platform/ingest/hydrolixlowerer.go | 71 +++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index e9d36ce5c..2ab107fc6 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -8,6 +8,7 @@ import ( "github.com/QuesmaOrg/quesma/platform/persistence" "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" + "strings" ) type HydrolixLowerer struct { @@ -20,37 +21,67 @@ func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixL } } -func (l *HydrolixLowerer) LowerToDDL(validatedJsons []types.JSON, +func (l *HydrolixLowerer) LowerToDDL( + validatedJsons []types.JSON, table *chLib.Table, invalidJsons []types.JSON, encodings map[schema.FieldEncodingKey]schema.EncodedFieldName, - createTableCmd CreateTableStatement) ([]string, error) { - for i, preprocessedJson := range validatedJsons { - _ = i - _ = preprocessedJson + createTableCmd CreateTableStatement, +) ([]string, error) { + // Construct columns array + var columnsJSON strings.Builder + columnsJSON.WriteString("[\n") + + for i, col := range createTableCmd.Columns { + if i > 0 { + columnsJSON.WriteString(",\n") + } + columnsJSON.WriteString(fmt.Sprintf(` { "name": "%s", "type": "%s"`, col.ColumnName, col.ColumnType)) + if col.Comment != "" { + columnsJSON.WriteString(fmt.Sprintf(`, "comment": "%s"`, col.Comment)) + } + if col.AdditionalMetadata != "" { + columnsJSON.WriteString(fmt.Sprintf(`, "metadata": "%s"`, col.AdditionalMetadata)) + } + columnsJSON.WriteString(" }") } - result := []string{fmt.Sprintf(`{ + columnsJSON.WriteString("\n]") + + const timeColumnName = "ingest_time" + + const ( + partitioningStrategy = "strategy" + partitioningField = "field" + partitioningGranularity = "granularity" + + defaultStrategy = "time" + defaultField = "ingest_time" + defaultGranularity = "day" + ) + partitioningJSON := fmt.Sprintf(`"partitioning": { + "%s": "%s", + "%s": "%s", + "%s": "%s" +}`, + partitioningStrategy, defaultStrategy, + partitioningField, defaultField, + partitioningGranularity, defaultGranularity) + + result := fmt.Sprintf(`{ "schema": { - "project": "", - "name": %s, - "time_column": "ingest_time", - "columns": [ - { "name": "new_field", "type": "string" }, - { "name": "ingest_time", "type": "datetime", "default": "NOW" } - ], - "partitioning": { - "strategy": "time", - "field": "ingest_time", - "granularity": "day" - } + "project": "%s", + "name": "%s", + "time_column": "%s", + "columns": %s, + %s, }, "events": [ { "new_field": "bar" } ] -}`, table.Name)} +}`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON) - return result, nil + return []string{result}, nil } From 6c0c356eec97e70393b4637967d9dbdfef610a51 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 27 Jun 2025 16:27:45 +0200 Subject: [PATCH 03/54] Moving some stuff around --- platform/ingest/hydrolixlowerer.go | 61 ++++++++++++++++++++++++++++++ platform/ingest/processor.go | 48 ----------------------- platform/ingest/sqllowerer.go | 49 ++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 48 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 2ab107fc6..2ef9576b3 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -9,10 +9,12 @@ import ( "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" "strings" + "sync/atomic" ) type HydrolixLowerer struct { virtualTableStorage persistence.JSONDatabase + ingestCounter int64 } func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixLowerer { @@ -21,6 +23,54 @@ func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixL } } +func (ip *HydrolixLowerer) GenerateIngestContent(table *chLib.Table, + data types.JSON, + inValidJson types.JSON, + encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) ([]AlterStatement, types.JSON, []NonSchemaField, error) { + + if len(table.Config.Attributes) == 0 { + return nil, data, nil, nil + } + + mDiff := DifferenceMap(data, table) // TODO change to DifferenceMap(m, t) + + if len(mDiff) == 0 && len(inValidJson) == 0 { // no need to modify, just insert 'js' + return nil, data, nil, nil + } + + // check attributes precondition + if len(table.Config.Attributes) <= 0 { + return nil, nil, nil, fmt.Errorf("no attributes config, but received non-schema fields: %s", mDiff) + } + attrsMap, _ := BuildAttrsMap(mDiff, table.Config) + + // generateNewColumns is called on original attributes map + // before adding invalid fields to it + // otherwise it would contain invalid fields e.g. with wrong types + // we only want to add fields that are not part of the schema e.g we don't + // have columns for them + var alterStatements []AlterStatement + atomic.AddInt64(&ip.ingestCounter, 1) + //if ok, alteredAttributesIndexes := ip.shouldAlterColumns(table, attrsMap); ok { + // alterStatements = ip.generateNewColumns(attrsMap, table, alteredAttributesIndexes, encodings) + //} + // If there are some invalid fields, we need to add them to the attributes map + // to not lose them and be able to store them later by + // generating correct update query + // addInvalidJsonFieldsToAttributes returns a new map with invalid fields added + // this map is then used to generate non-schema fields string + attrsMapWithInvalidFields := addInvalidJsonFieldsToAttributes(attrsMap, inValidJson) + nonSchemaFields, err := generateNonSchemaFields(attrsMapWithInvalidFields) + + if err != nil { + return nil, nil, nil, err + } + + onlySchemaFields := RemoveNonSchemaFields(data, table) + + return alterStatements, onlySchemaFields, nonSchemaFields, nil +} + func (l *HydrolixLowerer) LowerToDDL( validatedJsons []types.JSON, table *chLib.Table, @@ -68,6 +118,17 @@ func (l *HydrolixLowerer) LowerToDDL( partitioningField, defaultField, partitioningGranularity, defaultGranularity) + for i, preprocessedJson := range validatedJsons { + alter, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, + invalidJsons[i], encodings) + _ = alter + _ = onlySchemaFields + _ = nonSchemaFields + if err != nil { + return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) + } + } + result := fmt.Sprintf(`{ "schema": { "project": "%s", diff --git a/platform/ingest/processor.go b/platform/ingest/processor.go index b9cc1fd49..70cb506ea 100644 --- a/platform/ingest/processor.go +++ b/platform/ingest/processor.go @@ -627,54 +627,6 @@ func (ip *SqlLowerer) shouldAlterColumns(table *chLib.Table, attrsMap map[string return false, nil } -func (ip *SqlLowerer) GenerateIngestContent(table *chLib.Table, - data types.JSON, - inValidJson types.JSON, - encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) ([]AlterStatement, types.JSON, []NonSchemaField, error) { - - if len(table.Config.Attributes) == 0 { - return nil, data, nil, nil - } - - mDiff := DifferenceMap(data, table) // TODO change to DifferenceMap(m, t) - - if len(mDiff) == 0 && len(inValidJson) == 0 { // no need to modify, just insert 'js' - return nil, data, nil, nil - } - - // check attributes precondition - if len(table.Config.Attributes) <= 0 { - return nil, nil, nil, fmt.Errorf("no attributes config, but received non-schema fields: %s", mDiff) - } - attrsMap, _ := BuildAttrsMap(mDiff, table.Config) - - // generateNewColumns is called on original attributes map - // before adding invalid fields to it - // otherwise it would contain invalid fields e.g. with wrong types - // we only want to add fields that are not part of the schema e.g we don't - // have columns for them - var alterStatements []AlterStatement - atomic.AddInt64(&ip.ingestCounter, 1) - if ok, alteredAttributesIndexes := ip.shouldAlterColumns(table, attrsMap); ok { - alterStatements = ip.generateNewColumns(attrsMap, table, alteredAttributesIndexes, encodings) - } - // If there are some invalid fields, we need to add them to the attributes map - // to not lose them and be able to store them later by - // generating correct update query - // addInvalidJsonFieldsToAttributes returns a new map with invalid fields added - // this map is then used to generate non-schema fields string - attrsMapWithInvalidFields := addInvalidJsonFieldsToAttributes(attrsMap, inValidJson) - nonSchemaFields, err := generateNonSchemaFields(attrsMapWithInvalidFields) - - if err != nil { - return nil, nil, nil, err - } - - onlySchemaFields := RemoveNonSchemaFields(data, table) - - return alterStatements, onlySchemaFields, nonSchemaFields, nil -} - func generateInsertJson(nonSchemaFields []NonSchemaField, onlySchemaFields types.JSON) (string, error) { result := convertNonSchemaFieldsToMap(nonSchemaFields) diff --git a/platform/ingest/sqllowerer.go b/platform/ingest/sqllowerer.go index 8bb2b13ce..1160be8bf 100644 --- a/platform/ingest/sqllowerer.go +++ b/platform/ingest/sqllowerer.go @@ -10,6 +10,7 @@ import ( "github.com/QuesmaOrg/quesma/platform/types" "strings" "sync" + "sync/atomic" ) type SqlLowerer struct { @@ -26,6 +27,54 @@ func NewSqlLowerer(virtualTableStorage persistence.JSONDatabase) *SqlLowerer { } } +func (ip *SqlLowerer) GenerateIngestContent(table *chLib.Table, + data types.JSON, + inValidJson types.JSON, + encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) ([]AlterStatement, types.JSON, []NonSchemaField, error) { + + if len(table.Config.Attributes) == 0 { + return nil, data, nil, nil + } + + mDiff := DifferenceMap(data, table) // TODO change to DifferenceMap(m, t) + + if len(mDiff) == 0 && len(inValidJson) == 0 { // no need to modify, just insert 'js' + return nil, data, nil, nil + } + + // check attributes precondition + if len(table.Config.Attributes) <= 0 { + return nil, nil, nil, fmt.Errorf("no attributes config, but received non-schema fields: %s", mDiff) + } + attrsMap, _ := BuildAttrsMap(mDiff, table.Config) + + // generateNewColumns is called on original attributes map + // before adding invalid fields to it + // otherwise it would contain invalid fields e.g. with wrong types + // we only want to add fields that are not part of the schema e.g we don't + // have columns for them + var alterStatements []AlterStatement + atomic.AddInt64(&ip.ingestCounter, 1) + if ok, alteredAttributesIndexes := ip.shouldAlterColumns(table, attrsMap); ok { + alterStatements = ip.generateNewColumns(attrsMap, table, alteredAttributesIndexes, encodings) + } + // If there are some invalid fields, we need to add them to the attributes map + // to not lose them and be able to store them later by + // generating correct update query + // addInvalidJsonFieldsToAttributes returns a new map with invalid fields added + // this map is then used to generate non-schema fields string + attrsMapWithInvalidFields := addInvalidJsonFieldsToAttributes(attrsMap, inValidJson) + nonSchemaFields, err := generateNonSchemaFields(attrsMapWithInvalidFields) + + if err != nil { + return nil, nil, nil, err + } + + onlySchemaFields := RemoveNonSchemaFields(data, table) + + return alterStatements, onlySchemaFields, nonSchemaFields, nil +} + func (l *SqlLowerer) LowerToDDL(validatedJsons []types.JSON, table *chLib.Table, invalidJsons []types.JSON, From 11c7f46a9b9c9d6181ef4a05d6eb75981fbd8521 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 30 Jun 2025 12:05:00 +0200 Subject: [PATCH 04/54] Populate events --- platform/ingest/hydrolixlowerer.go | 36 +++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 2ef9576b3..8e5455ccd 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -3,6 +3,7 @@ package ingest import ( + "encoding/json" "fmt" chLib "github.com/QuesmaOrg/quesma/platform/clickhouse" "github.com/QuesmaOrg/quesma/platform/persistence" @@ -117,17 +118,33 @@ func (l *HydrolixLowerer) LowerToDDL( partitioningStrategy, defaultStrategy, partitioningField, defaultField, partitioningGranularity, defaultGranularity) - + events := make(map[string]any) for i, preprocessedJson := range validatedJsons { - alter, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, + _, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, invalidJsons[i], encodings) - _ = alter - _ = onlySchemaFields - _ = nonSchemaFields if err != nil { return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) } + if err != nil { + return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) + } + content := convertNonSchemaFieldsToMap(nonSchemaFields) + + for k, v := range onlySchemaFields { + content[k] = v + } + + for k, v := range content { + events[k] = v + } + } + + eventList := []map[string]any{events} + eventBytes, err := json.MarshalIndent(eventList, " ", " ") + if err != nil { + return nil, err } + eventJSON := string(eventBytes) result := fmt.Sprintf(`{ "schema": { @@ -137,12 +154,9 @@ func (l *HydrolixLowerer) LowerToDDL( "columns": %s, %s, }, - "events": [ - { - "new_field": "bar" - } - ] -}`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON) + "events": %s +}`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON, eventJSON) + fmt.Println(result) return []string{result}, nil } From af2c0544f33ef9be14cf5d6d7965012014badd52 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 30 Jun 2025 12:35:29 +0200 Subject: [PATCH 05/54] Update test and Exec method --- .../hydrolix_backend_connector.go | 7 ++++- platform/ingest/hydrolixlowerer.go | 2 -- platform/ingest/insert_test.go | 26 ++++++++++--------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 118e49520..b790a1663 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -48,5 +48,10 @@ func (p *HydrolixBackendConnector) InstanceName() string { } func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { - return nil + if len(args) == 0 { + _, err := p.connection.ExecContext(ctx, query) + return err + } + _, err := p.connection.ExecContext(ctx, query, args...) + return err } diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 8e5455ccd..377086c84 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -156,7 +156,5 @@ func (l *HydrolixLowerer) LowerToDDL( }, "events": %s }`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON, eventJSON) - - fmt.Println(result) return []string{result}, nil } diff --git a/platform/ingest/insert_test.go b/platform/ingest/insert_test.go index c3230e583..e3298c281 100644 --- a/platform/ingest/insert_test.go +++ b/platform/ingest/insert_test.go @@ -480,25 +480,27 @@ func TestHydrolixIngest(t *testing.T) { expectedStatements: []string{ fmt.Sprintf(`{ - "schema": { +"schema": { "project": "%s", "name": "test_index", "time_column": "ingest_time", "columns": [ - { "name": "new_field", "type": "string" }, - { "name": "ingest_time", "type": "datetime", "default": "NOW" } - ], + { "name": "@timestamp", "type": "DateTime64(3)", "metadata": "DEFAULT now64()" }, + { "name": "attributes_values", "type": "Map(String,String)" }, + { "name": "attributes_metadata", "type": "Map(String,String)" }, + { "name": "new_field", "type": "Nullable(String)", "comment": "quesmaMetadataV1:fieldName=new_field" } +], "partitioning": { - "strategy": "time", - "field": "ingest_time", - "granularity": "day" - } + "strategy": "time", + "field": "ingest_time", + "granularity": "day" +}, }, "events": [ - { - "new_field": "bar" - } - ] + { + "new_field": "bar" + } + ] }`, projectName), }, }, From bddae1a7ec17fd9dacb71f54aeccbe72e3370a03 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 30 Jun 2025 13:44:41 +0200 Subject: [PATCH 06/54] HydrolixBackendConnector Exec --- .../hydrolix_backend_connector.go | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index b790a1663..ca4649fd3 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -4,16 +4,23 @@ package backend_connectors import ( + "bytes" "context" "database/sql" + "fmt" "github.com/QuesmaOrg/quesma/platform/config" + "github.com/QuesmaOrg/quesma/platform/logger" + "io" + "net/http" quesma_api "github.com/QuesmaOrg/quesma/platform/v2/core" ) type HydrolixBackendConnector struct { BasicSqlBackendConnector - cfg *config.RelationalDbConfiguration + cfg *config.RelationalDbConfiguration + IngestURL string + AccessToken string } func (p *HydrolixBackendConnector) GetId() quesma_api.BackendConnectorType { @@ -48,10 +55,34 @@ func (p *HydrolixBackendConnector) InstanceName() string { } func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { - if len(args) == 0 { + if p.IngestURL == "" || p.AccessToken == "" { + logger.Info().Msg("missing ingest URL or access token") + // TODO for fallback, execute the query directly on the database connection _, err := p.connection.ExecContext(ctx, query) return err } - _, err := p.connection.ExecContext(ctx, query, args...) - return err + + // Create HTTP request using the JSON payload from query + req, err := http.NewRequestWithContext(ctx, "POST", p.IngestURL, bytes.NewBufferString(query)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+p.AccessToken) + req.Header.Set("Content-Type", "application/json") + + // Execute HTTP request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + // Handle error response + if resp.StatusCode >= 400 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("ingest failed: %s — %s", resp.Status, string(body)) + } + + return nil } From 06236ac64400b3a632f3a9ba68d5ee078b6ba906 Mon Sep 17 00:00:00 2001 From: Przemyslaw Delewski <102958445+pdelewski@users.noreply.github.com> Date: Thu, 3 Jul 2025 13:35:33 +0200 Subject: [PATCH 07/54] Update platform/ingest/hydrolixlowerer.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Przemysław Hejman Signed-off-by: Przemyslaw Delewski <102958445+pdelewski@users.noreply.github.com> --- platform/ingest/hydrolixlowerer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 377086c84..ba426842a 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -26,7 +26,7 @@ func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixL func (ip *HydrolixLowerer) GenerateIngestContent(table *chLib.Table, data types.JSON, - inValidJson types.JSON, + invalidJson types.JSON, encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) ([]AlterStatement, types.JSON, []NonSchemaField, error) { if len(table.Config.Attributes) == 0 { From b5461b6b0b3dfcfdebd2e567ddf82f0a6fdabc9d Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 4 Jul 2025 14:06:46 +0200 Subject: [PATCH 08/54] Fix compilation error --- platform/ingest/hydrolixlowerer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index ba426842a..377086c84 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -26,7 +26,7 @@ func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixL func (ip *HydrolixLowerer) GenerateIngestContent(table *chLib.Table, data types.JSON, - invalidJson types.JSON, + inValidJson types.JSON, encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) ([]AlterStatement, types.JSON, []NonSchemaField, error) { if len(table.Config.Attributes) == 0 { From 6426c82eb9a9339543cad76e84d0ffd247dc99c5 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 4 Jul 2025 14:10:43 +0200 Subject: [PATCH 09/54] Fixing linter --- platform/ingest/processor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/platform/ingest/processor.go b/platform/ingest/processor.go index 9058ff494..a9ed1570c 100644 --- a/platform/ingest/processor.go +++ b/platform/ingest/processor.go @@ -627,7 +627,6 @@ func (ip *SqlLowerer) shouldAlterColumns(table *database_common.Table, attrsMap return false, nil } - func generateInsertJson(nonSchemaFields []NonSchemaField, onlySchemaFields types.JSON) (string, error) { result := convertNonSchemaFieldsToMap(nonSchemaFields) From f0da77ea7fe917e77bf45f8061ce724069624c6c Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 4 Jul 2025 14:29:11 +0200 Subject: [PATCH 10/54] Changing to atomic --- platform/ingest/hydrolixlowerer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index c466b5712..5d8f77d83 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -15,7 +15,7 @@ import ( type HydrolixLowerer struct { virtualTableStorage persistence.JSONDatabase - ingestCounter int64 + ingestCounter atomic.Int64 } func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixLowerer { @@ -51,7 +51,7 @@ func (ip *HydrolixLowerer) GenerateIngestContent(table *chLib.Table, // we only want to add fields that are not part of the schema e.g we don't // have columns for them var alterStatements []AlterStatement - atomic.AddInt64(&ip.ingestCounter, 1) + ip.ingestCounter.Add(1) //if ok, alteredAttributesIndexes := ip.shouldAlterColumns(table, attrsMap); ok { // alterStatements = ip.generateNewColumns(attrsMap, table, alteredAttributesIndexes, encodings) //} From 6bc5ba36edb20517cdfee6f4233202144d00be42 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 4 Jul 2025 14:32:11 +0200 Subject: [PATCH 11/54] Adding comment --- platform/backend_connectors/hydrolix_backend_connector.go | 1 + 1 file changed, 1 insertion(+) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index ca4649fd3..8df5338fb 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -18,6 +18,7 @@ import ( type HydrolixBackendConnector struct { BasicSqlBackendConnector + // TODO for now we still have reference for RelationalDbConfiguration for fallback cfg *config.RelationalDbConfiguration IngestURL string AccessToken string From cf8673c215070d931406a14b1aa73d4f5b9e93aa Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 4 Jul 2025 15:04:43 +0200 Subject: [PATCH 12/54] Json sanity check --- .../backend_connectors/hydrolix_backend_connector.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 8df5338fb..2870c3664 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "database/sql" + "encoding/json" "fmt" "github.com/QuesmaOrg/quesma/platform/config" "github.com/QuesmaOrg/quesma/platform/logger" @@ -55,6 +56,11 @@ func (p *HydrolixBackendConnector) InstanceName() string { return "hydrolix" // TODO add name taken from config } +func isValidJSON(s string) bool { + var js interface{} + return json.Unmarshal([]byte(s), &js) == nil +} + func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { if p.IngestURL == "" || p.AccessToken == "" { logger.Info().Msg("missing ingest URL or access token") @@ -63,6 +69,10 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args return err } + if !isValidJSON(query) { + return fmt.Errorf("invalid JSON payload: %s", query) + } + // Create HTTP request using the JSON payload from query req, err := http.NewRequestWithContext(ctx, "POST", p.IngestURL, bytes.NewBufferString(query)) if err != nil { From 89b73ffbb67ee889a7bb36035ce249c3399a1f50 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 15 Jul 2025 13:31:45 +0200 Subject: [PATCH 13/54] Use hydrolix backend connector --- .../backend_connectors/hydrolix_backend_connector.go | 9 ++++++++- platform/clickhouse/connection.go | 4 ++++ platform/licensing/runner.go | 3 ++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 2870c3664..b25b7210d 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -6,6 +6,7 @@ package backend_connectors import ( "bytes" "context" + "crypto/tls" "database/sql" "encoding/json" "fmt" @@ -23,6 +24,7 @@ type HydrolixBackendConnector struct { cfg *config.RelationalDbConfiguration IngestURL string AccessToken string + Headers map[string]string } func (p *HydrolixBackendConnector) GetId() quesma_api.BackendConnectorType { @@ -79,10 +81,15 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args return fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Authorization", "Bearer "+p.AccessToken) + req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") // Execute HTTP request - client := &http.Client{} + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } resp, err := client.Do(req) if err != nil { return fmt.Errorf("request failed: %w", err) diff --git a/platform/clickhouse/connection.go b/platform/clickhouse/connection.go index 44b1a08df..fc6e59ea8 100644 --- a/platform/clickhouse/connection.go +++ b/platform/clickhouse/connection.go @@ -97,7 +97,11 @@ func InitDBConnectionPool(c *config.QuesmaConfiguration) quesma_api.BackendConne // clean up connections after 5 minutes, before that they may be killed by the firewall db.SetConnMaxLifetime(time.Duration(5) * time.Minute) // default is 1h + if c.Hydrolix.ConnectorType == quesma_api.GetBackendConnectorNameFromType(quesma_api.HydrolixSQLBackend) { + return backend_connectors.NewHydrolixBackendConnectorWithConnection(c.Hydrolix.Url.String(), db) + } return backend_connectors.NewClickHouseBackendConnectorWithConnection(c.ClickHouse.Url.String(), db) + } // RunClickHouseConnectionDoctor is very blunt and verbose function which aims to print some helpful information diff --git a/platform/licensing/runner.go b/platform/licensing/runner.go index 37c48b812..021273870 100644 --- a/platform/licensing/runner.go +++ b/platform/licensing/runner.go @@ -82,7 +82,8 @@ func (l *LicenseModule) validateConfig() error { // Check if connectors are allowed for _, conn := range l.Config.Connectors { if !slices.Contains(l.License.Connectors, conn.ConnectorType) { - return fmt.Errorf("connector of type [%s] is not allowed within the current license", conn.ConnectorType) + // TODO !!!!! + //return fmt.Errorf("connector of type [%s] is not allowed within the current license", conn.ConnectorType) } } return nil From 6b24e67fd72fc3fa7d9285e10c0cf616ee25ba3a Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 15 Jul 2025 17:38:15 +0200 Subject: [PATCH 14/54] First ingest (still hardcoded) to hydrolix via quesma --- .../hydrolix_backend_connector.go | 123 +++++++-- platform/ingest/hydrolixlowerer.go | 235 ++++++++++++------ 2 files changed, 256 insertions(+), 102 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index b25b7210d..0a76589ee 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -12,10 +12,11 @@ import ( "fmt" "github.com/QuesmaOrg/quesma/platform/config" "github.com/QuesmaOrg/quesma/platform/logger" + quesma_api "github.com/QuesmaOrg/quesma/platform/v2/core" + "github.com/google/uuid" "io" "net/http" - - quesma_api "github.com/QuesmaOrg/quesma/platform/v2/core" + "time" ) type HydrolixBackendConnector struct { @@ -63,43 +64,119 @@ func isValidJSON(s string) bool { return json.Unmarshal([]byte(s), &js) == nil } -func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { - if p.IngestURL == "" || p.AccessToken == "" { - logger.Info().Msg("missing ingest URL or access token") - // TODO for fallback, execute the query directly on the database connection - _, err := p.connection.ExecContext(ctx, query) - return err - } - - if !isValidJSON(query) { - return fmt.Errorf("invalid JSON payload: %s", query) - } - - // Create HTTP request using the JSON payload from query - req, err := http.NewRequestWithContext(ctx, "POST", p.IngestURL, bytes.NewBufferString(query)) +func makeRequest(ctx context.Context, method string, url string, body []byte, token string, tableName string) (error, []byte) { + // Build the request + req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body)) if err != nil { - return fmt.Errorf("failed to create request: %w", err) + panic(err) } - req.Header.Set("Authorization", "Bearer "+p.AccessToken) + + // Set headers + req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-hdx-table", "sample_project."+tableName) - // Execute HTTP request + // Allow self-signed certs (equivalent to curl -k) client := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } + + // Send the request resp, err := client.Do(req) if err != nil { - return fmt.Errorf("request failed: %w", err) + panic(err) } defer resp.Body.Close() - // Handle error response + // Read and print response + respBody, err := io.ReadAll(resp.Body) if resp.StatusCode >= 400 { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("ingest failed: %s — %s", resp.Status, string(body)) + return fmt.Errorf("ingest failed: %s — %s", resp.Status, string(respBody)), nil + } + return err, respBody +} + +var tableId uuid.UUID +var transformCreated bool +var tableName string + +func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { + token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI2NTgxNDYsImlhdCI6MTc1MjU3MTc0NiwianRpIjoiODI2ZTViNjgtNWM4MS00NTUxLWI3N2EtOTZkNmVkNTM2ZTA1IiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiYWRlZTZjN2UtZmM4Yi00NzY4LTk5NTktY2FkY2Q3YWM5M2RjIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiYWRlZTZjN2UtZmM4Yi00NzY4LTk5NTktY2FkY2Q3YWM5M2RjIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.F2NIUxD0lD-g5Q729jU5JN_ZfGQqvVZcHle3scMrG529czTfBZUoOBb90YFxPhHU7owzomTJ7v077UMTfgGuRUZNHy9CX2G33UB9Fy7RllgK-eW1MyFKxNdEDVkE-gESG7wtHqd-mxG_Yt9plXJs1wVHtrnYJ9GxKaWzpWaCMfFKq-rr6A9Ghuzr-FgWOvgTot9CExR8ThdOwVZREWXxhG0ki3bTnqQ1GRpwstORFPsPdJrtNaubZrGfqyjclMpLWRv4OVkDxQkAeW5ZcrjbtjdIed8Y1NIiWju74iOditHU4BiIfK82R8TlN112qMx8KjKq1gScpgjK8Jo6VKhrFA" + hdxHost := "3.20.203.177:8888" + orgID := "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" + projectID := "27506b30-0c78-41fa-a059-048d687f1164" + + if !isValidJSON(query) { + return fmt.Errorf("invalid JSON payload: %s", query) + } + + // Top-level object + var root map[string]json.RawMessage + if err := json.Unmarshal([]byte(query), &root); err != nil { + panic(err) + } + + // Extract each section into its own map (or struct, if needed) + var createTable map[string]interface{} + var transform map[string]interface{} + var ingest map[string]interface{} + + if err := json.Unmarshal(root["create_table"], &createTable); err != nil { + panic(err) + } + if err := json.Unmarshal(root["transform"], &transform); err != nil { + panic(err) + } + if err := json.Unmarshal(root["ingest"], &ingest); err != nil { + panic(err) + } + + if len(createTable) > 0 && tableId == uuid.Nil { + url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) + tableName = createTable["name"].(string) + if tableId == uuid.Nil { + tableId = uuid.New() + } + createTable["uuid"] = tableId.String() + createTableJson, err := json.Marshal(createTable) + if err != nil { + return fmt.Errorf("error marshalling create_table JSON: %v", err) + } + err, _ = makeRequest(ctx, "POST", url, createTableJson, token, tableName) + if err != nil { + logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) + return err + } + } + if len(transform) > 0 && !transformCreated { + url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", hdxHost, orgID, projectID, tableId.String()) + transformJson, err := json.Marshal(transform) + if err != nil { + return fmt.Errorf("error marshalling transform JSON: %v", err) + } + + err, _ = makeRequest(ctx, "POST", url, transformJson, token, tableName) + if err != nil { + logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) + return err + } + time.Sleep(5 * time.Second) // Wait for the transform to be created + transformCreated = true + } + if len(ingest) > 0 && transformCreated { + ingestJson, err := json.Marshal(ingest) + if err != nil { + return fmt.Errorf("error marshalling ingest JSON: %v", err) + } + url := fmt.Sprintf("http://%s/ingest/event", hdxHost) + err, _ = makeRequest(ctx, "POST", url, ingestJson, token, tableName) + if err != nil { + logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) + return err + } } return nil diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 5d8f77d83..f06eea063 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -3,13 +3,11 @@ package ingest import ( - "encoding/json" "fmt" chLib "github.com/QuesmaOrg/quesma/platform/database_common" "github.com/QuesmaOrg/quesma/platform/persistence" "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" - "strings" "sync/atomic" ) @@ -79,82 +77,161 @@ func (l *HydrolixLowerer) LowerToDDL( encodings map[schema.FieldEncodingKey]schema.EncodedFieldName, createTableCmd CreateTableStatement, ) ([]string, error) { - // Construct columns array - var columnsJSON strings.Builder - columnsJSON.WriteString("[\n") - - for i, col := range createTableCmd.Columns { - if i > 0 { - columnsJSON.WriteString(",\n") - } - columnsJSON.WriteString(fmt.Sprintf(` { "name": "%s", "type": "%s"`, col.ColumnName, col.ColumnType)) - if col.Comment != "" { - columnsJSON.WriteString(fmt.Sprintf(`, "comment": "%s"`, col.Comment)) - } - if col.AdditionalMetadata != "" { - columnsJSON.WriteString(fmt.Sprintf(`, "metadata": "%s"`, col.AdditionalMetadata)) - } - columnsJSON.WriteString(" }") - } - - columnsJSON.WriteString("\n]") - - const timeColumnName = "ingest_time" - - const ( - partitioningStrategy = "strategy" - partitioningField = "field" - partitioningGranularity = "granularity" - - defaultStrategy = "time" - defaultField = "ingest_time" - defaultGranularity = "day" - ) - partitioningJSON := fmt.Sprintf(`"partitioning": { - "%s": "%s", - "%s": "%s", - "%s": "%s" -}`, - partitioningStrategy, defaultStrategy, - partitioningField, defaultField, - partitioningGranularity, defaultGranularity) - events := make(map[string]any) - for i, preprocessedJson := range validatedJsons { - _, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, - invalidJsons[i], encodings) - if err != nil { - return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) - } - if err != nil { - return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) - } - content := convertNonSchemaFieldsToMap(nonSchemaFields) - - for k, v := range onlySchemaFields { - content[k] = v - } - - for k, v := range content { - events[k] = v - } - } + /* + // Construct columns array + var columnsJSON strings.Builder + columnsJSON.WriteString("[\n") + + for i, col := range createTableCmd.Columns { + if i > 0 { + columnsJSON.WriteString(",\n") + } + columnsJSON.WriteString(fmt.Sprintf(` { "name": "%s", "type": "%s"`, col.ColumnName, col.ColumnType)) + if col.Comment != "" { + columnsJSON.WriteString(fmt.Sprintf(`, "comment": "%s"`, col.Comment)) + } + if col.AdditionalMetadata != "" { + columnsJSON.WriteString(fmt.Sprintf(`, "metadata": "%s"`, col.AdditionalMetadata)) + } + columnsJSON.WriteString(" }") + } + + columnsJSON.WriteString("\n]") + + const timeColumnName = "ingest_time" + + const ( + partitioningStrategy = "strategy" + partitioningField = "field" + partitioningGranularity = "granularity" + + defaultStrategy = "time" + defaultField = "ingest_time" + defaultGranularity = "day" + ) + partitioningJSON := fmt.Sprintf(`"partitioning": { + "%s": "%s", + "%s": "%s", + "%s": "%s" + }`, + partitioningStrategy, defaultStrategy, + partitioningField, defaultField, + partitioningGranularity, defaultGranularity) + events := make(map[string]any) + for i, preprocessedJson := range validatedJsons { + _, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, + invalidJsons[i], encodings) + if err != nil { + return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) + } + if err != nil { + return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) + } + content := convertNonSchemaFieldsToMap(nonSchemaFields) + + for k, v := range onlySchemaFields { + content[k] = v + } + + for k, v := range content { + events[k] = v + } + } + + eventList := []map[string]any{events} + eventBytes, err := json.MarshalIndent(eventList, " ", " ") + if err != nil { + return nil, err + } + eventJSON := string(eventBytes) + + result := fmt.Sprintf(`{ + "schema": { + "project": "%s", + "name": "%s", + "time_column": "%s", + "columns": %s, + %s, + }, + "events": %s + }`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON, eventJSON) + return []string{result}, nil + */ + tableName := "pdelewski43" + + ingestBody := []byte(fmt.Sprintf(`{ + "create_table" : { + "name": "%s", + "settings": { + "merge": { + "enabled": true + } + } + }, + "transform": { + "name" : "transform1", + "type": "json", + "settings": { + "format_details": { + "flattening": { "active": false } + }, + "output_columns": [ + { + "name": "timestamp", + "datatype": { + "type": "datetime", + "primary": true, + "format": "2006-01-02 15:04:05 MST" + } + }, + { + "name": "clientId", + "datatype": { + "type": "uint64" + } + }, + { + "name": "clientIp", + "datatype": { + "type": "string", + "index": true, + "default": "0.0.0.0" + } + }, + { + "name": "clientCityCode", + "datatype": { + "type": "uint32" + } + }, + { + "name": "resolverIp", + "datatype": { + "type": "string", + "index": true, + "default": "0.0.0.0" + } + }, + { + "name": "resolveDuration", + "datatype": { + "type": "double", + "default": -1.0 + } + } + ] + } + }, + "ingest": { + "timestamp": "2020-02-26 16:01:27 PST", + "clientId": "29992", + "clientIp": "1.2.3.4/24", + "clientCityCode": 1224, + "resolverIp": "1.4.5.7", + "resolveDuration": "1.234" + } + }`, tableName)) + + return []string{string(ingestBody)}, nil - eventList := []map[string]any{events} - eventBytes, err := json.MarshalIndent(eventList, " ", " ") - if err != nil { - return nil, err - } - eventJSON := string(eventBytes) - - result := fmt.Sprintf(`{ - "schema": { - "project": "%s", - "name": "%s", - "time_column": "%s", - "columns": %s, - %s, - }, - "events": %s -}`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON, eventJSON) - return []string{result}, nil } From 2910aebae9dfe95061c3e86315cf28fc0e4239a3 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 15 Jul 2025 18:28:57 +0200 Subject: [PATCH 15/54] Use maps instead of strings --- platform/ingest/hydrolixlowerer.go | 172 ++++++++++++++++------------- 1 file changed, 96 insertions(+), 76 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index f06eea063..d882177c5 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -8,6 +8,7 @@ import ( "github.com/QuesmaOrg/quesma/platform/persistence" "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" + "github.com/goccy/go-json" "sync/atomic" ) @@ -157,81 +158,100 @@ func (l *HydrolixLowerer) LowerToDDL( }`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON, eventJSON) return []string{result}, nil */ - tableName := "pdelewski43" - - ingestBody := []byte(fmt.Sprintf(`{ - "create_table" : { - "name": "%s", - "settings": { - "merge": { - "enabled": true - } - } - }, - "transform": { - "name" : "transform1", - "type": "json", - "settings": { - "format_details": { - "flattening": { "active": false } - }, - "output_columns": [ - { - "name": "timestamp", - "datatype": { - "type": "datetime", - "primary": true, - "format": "2006-01-02 15:04:05 MST" - } - }, - { - "name": "clientId", - "datatype": { - "type": "uint64" - } - }, - { - "name": "clientIp", - "datatype": { - "type": "string", - "index": true, - "default": "0.0.0.0" - } - }, - { - "name": "clientCityCode", - "datatype": { - "type": "uint32" - } - }, - { - "name": "resolverIp", - "datatype": { - "type": "string", - "index": true, - "default": "0.0.0.0" - } - }, - { - "name": "resolveDuration", - "datatype": { - "type": "double", - "default": -1.0 - } - } - ] - } - }, - "ingest": { - "timestamp": "2020-02-26 16:01:27 PST", - "clientId": "29992", - "clientIp": "1.2.3.4/24", - "clientCityCode": 1224, - "resolverIp": "1.4.5.7", - "resolveDuration": "1.234" - } - }`, tableName)) - - return []string{string(ingestBody)}, nil + tableName := "pdelewski44" + + // --- Create Table Section --- + createTable := map[string]interface{}{ + "name": tableName, + "settings": map[string]interface{}{ + "merge": map[string]interface{}{ + "enabled": true, + }, + }, + } + + // --- Output Columns Slice --- + outputColumns := []interface{}{ + map[string]interface{}{ + "name": "timestamp", + "datatype": map[string]interface{}{ + "type": "datetime", + "primary": true, + "format": "2006-01-02 15:04:05 MST", + }, + }, + map[string]interface{}{ + "name": "clientId", + "datatype": map[string]interface{}{ + "type": "uint64", + }, + }, + map[string]interface{}{ + "name": "clientIp", + "datatype": map[string]interface{}{ + "type": "string", + "index": true, + "default": "0.0.0.0", + }, + }, + map[string]interface{}{ + "name": "clientCityCode", + "datatype": map[string]interface{}{ + "type": "uint32", + }, + }, + map[string]interface{}{ + "name": "resolverIp", + "datatype": map[string]interface{}{ + "type": "string", + "index": true, + "default": "0.0.0.0", + }, + }, + map[string]interface{}{ + "name": "resolveDuration", + "datatype": map[string]interface{}{ + "type": "double", + "default": -1.0, + }, + }, + } + + // --- Transform Section --- + transform := map[string]interface{}{ + "name": "transform1", + "type": "json", + "settings": map[string]interface{}{ + "format_details": map[string]interface{}{ + "flattening": map[string]interface{}{ + "active": false, + }, + }, + "output_columns": outputColumns, + }, + } + + // --- Ingest Section --- + ingest := map[string]interface{}{ + "timestamp": "2020-02-26 16:01:27 PST", + "clientId": "29992", + "clientIp": "1.2.3.4/24", + "clientCityCode": 1224, + "resolverIp": "1.4.5.7", + "resolveDuration": "1.234", + } + + // --- Final Payload --- + payload := map[string]interface{}{ + "create_table": createTable, + "transform": transform, + "ingest": ingest, + } + + marshaledPayload, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("error marshalling payload: %v", err) + } + return []string{string(marshaledPayload)}, nil } From 32dd1c5109882304197611fc76e62d667c81c75a Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 15 Jul 2025 18:34:06 +0200 Subject: [PATCH 16/54] Remove hardcoded name --- platform/ingest/hydrolixlowerer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index d882177c5..4296eb973 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -158,11 +158,10 @@ func (l *HydrolixLowerer) LowerToDDL( }`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON, eventJSON) return []string{result}, nil */ - tableName := "pdelewski44" // --- Create Table Section --- createTable := map[string]interface{}{ - "name": tableName, + "name": table.Name, "settings": map[string]interface{}{ "merge": map[string]interface{}{ "enabled": true, From 57c87e1923ca2ef4174d226558ae6b0aec087b18 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 12:28:49 +0200 Subject: [PATCH 17/54] Handle more than one table --- .../hydrolix_backend_connector.go | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 0a76589ee..1214eab80 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -16,6 +16,7 @@ import ( "github.com/google/uuid" "io" "net/http" + "sync" "time" ) @@ -100,11 +101,12 @@ func makeRequest(ctx context.Context, method string, url string, body []byte, to } var tableId uuid.UUID -var transformCreated bool var tableName string +var tableCache = make(map[string]uuid.UUID) +var tableMutex sync.Mutex func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { - token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI2NTgxNDYsImlhdCI6MTc1MjU3MTc0NiwianRpIjoiODI2ZTViNjgtNWM4MS00NTUxLWI3N2EtOTZkNmVkNTM2ZTA1IiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiYWRlZTZjN2UtZmM4Yi00NzY4LTk5NTktY2FkY2Q3YWM5M2RjIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiYWRlZTZjN2UtZmM4Yi00NzY4LTk5NTktY2FkY2Q3YWM5M2RjIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.F2NIUxD0lD-g5Q729jU5JN_ZfGQqvVZcHle3scMrG529czTfBZUoOBb90YFxPhHU7owzomTJ7v077UMTfgGuRUZNHy9CX2G33UB9Fy7RllgK-eW1MyFKxNdEDVkE-gESG7wtHqd-mxG_Yt9plXJs1wVHtrnYJ9GxKaWzpWaCMfFKq-rr6A9Ghuzr-FgWOvgTot9CExR8ThdOwVZREWXxhG0ki3bTnqQ1GRpwstORFPsPdJrtNaubZrGfqyjclMpLWRv4OVkDxQkAeW5ZcrjbtjdIed8Y1NIiWju74iOditHU4BiIfK82R8TlN112qMx8KjKq1gScpgjK8Jo6VKhrFA" + token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI3NDUyMjYsImlhdCI6MTc1MjY1ODgyNiwianRpIjoiMjE1NGQ2YTItNTBiOC00MWM2LTk4MDYtYzE5N2M4YTUzZjJlIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiMzA0NTg5MTItMjkzYS00MDUwLWEwZDMtNmExMzc0ZjkzZTBhIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiMzA0NTg5MTItMjkzYS00MDUwLWEwZDMtNmExMzc0ZjkzZTBhIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.OAqqHSA2UePYIryGVvF4AFhxxTnAbdLa-xnmAWJUoECkZnGlLWdkRYemIq2aoTc2KgWJwa_afh1sXMpXafpfQm_dFNDVk7dAgQMTOfSM8RyrcSJBJTTQDNFriPAmRsfQW4al33tKPj4cr527x00oDs8U8dRQYX0t2c2ae74YTZ1ncE8AQrA48dpBNf9wHjjNxX1m1TQk_129Wvc0BG0QtX3yiir6FtktbBGhDQFSFS-OYdGItFDrfKsMaB7yN3X4GrNU9yv4ZgYUEEHWpU6SL8ATzDX9hWBpotn1svY9BT99N4bvC5iRVeAoOs4dlmv7Ctydgaw8NSSy68SVO6ztfg" hdxHost := "3.20.203.177:8888" orgID := "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" projectID := "27506b30-0c78-41fa-a059-048d687f1164" @@ -134,12 +136,19 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args panic(err) } + // Check if tableId is already cached + tableMutex.Lock() + if id, exists := tableCache[createTable["name"].(string)]; exists { + tableId = id + } else { + tableId = uuid.Nil + } + tableMutex.Unlock() + if len(createTable) > 0 && tableId == uuid.Nil { url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) tableName = createTable["name"].(string) - if tableId == uuid.Nil { - tableId = uuid.New() - } + tableId = uuid.New() createTable["uuid"] = tableId.String() createTableJson, err := json.Marshal(createTable) if err != nil { @@ -150,9 +159,8 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err } - } - if len(transform) > 0 && !transformCreated { - url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", hdxHost, orgID, projectID, tableId.String()) + + url = fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", hdxHost, orgID, projectID, tableId.String()) transformJson, err := json.Marshal(transform) if err != nil { return fmt.Errorf("error marshalling transform JSON: %v", err) @@ -164,9 +172,8 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args return err } time.Sleep(5 * time.Second) // Wait for the transform to be created - transformCreated = true } - if len(ingest) > 0 && transformCreated { + if len(ingest) > 0 && tableId != uuid.Nil { ingestJson, err := json.Marshal(ingest) if err != nil { return fmt.Errorf("error marshalling ingest JSON: %v", err) From 3e54942ba890bcb6b85271a1fddac6c490e80d67 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 13:02:26 +0200 Subject: [PATCH 18/54] Schema improvements, almost correct column names --- platform/ingest/hydrolixlowerer.go | 54 ++++++++---------------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 4296eb973..dc3c516bc 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -170,50 +170,24 @@ func (l *HydrolixLowerer) LowerToDDL( } // --- Output Columns Slice --- - outputColumns := []interface{}{ - map[string]interface{}{ - "name": "timestamp", - "datatype": map[string]interface{}{ - "type": "datetime", - "primary": true, - "format": "2006-01-02 15:04:05 MST", - }, + outputColumns := make([]interface{}, 0, len(createTableCmd.Columns)) + outputColumns = append(outputColumns, map[string]interface{}{ + "name": "__timestamp", + "datatype": map[string]interface{}{ + "type": "datetime", + "primary": true, + "format": "2006-01-02 15:04:05 MST", }, - map[string]interface{}{ - "name": "clientId", + }) + for _, col := range createTableCmd.Columns { + columnMap := map[string]interface{}{ + "name": col.ColumnName, "datatype": map[string]interface{}{ "type": "uint64", }, - }, - map[string]interface{}{ - "name": "clientIp", - "datatype": map[string]interface{}{ - "type": "string", - "index": true, - "default": "0.0.0.0", - }, - }, - map[string]interface{}{ - "name": "clientCityCode", - "datatype": map[string]interface{}{ - "type": "uint32", - }, - }, - map[string]interface{}{ - "name": "resolverIp", - "datatype": map[string]interface{}{ - "type": "string", - "index": true, - "default": "0.0.0.0", - }, - }, - map[string]interface{}{ - "name": "resolveDuration", - "datatype": map[string]interface{}{ - "type": "double", - "default": -1.0, - }, - }, + } + + outputColumns = append(outputColumns, columnMap) } // --- Transform Section --- From 0382627f19956caed4db8e6e4e7caceded947e99 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 18:04:15 +0200 Subject: [PATCH 19/54] Handle arrays and maps #1 --- platform/ingest/hydrolixlowerer.go | 89 +++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 13 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index dc3c516bc..ba0bcacbb 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -9,6 +9,8 @@ import ( "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" "github.com/goccy/go-json" + "regexp" + "strings" "sync/atomic" ) @@ -71,6 +73,11 @@ func (ip *HydrolixLowerer) GenerateIngestContent(table *chLib.Table, return alterStatements, onlySchemaFields, nonSchemaFields, nil } +func unwrapNullable(input string) string { + re := regexp.MustCompile(`Nullable\(([^)]+)\)`) + return re.ReplaceAllString(input, `$1`) +} + func (l *HydrolixLowerer) LowerToDDL( validatedJsons []types.JSON, table *chLib.Table, @@ -170,21 +177,77 @@ func (l *HydrolixLowerer) LowerToDDL( } // --- Output Columns Slice --- - outputColumns := make([]interface{}, 0, len(createTableCmd.Columns)) - outputColumns = append(outputColumns, map[string]interface{}{ - "name": "__timestamp", - "datatype": map[string]interface{}{ - "type": "datetime", - "primary": true, - "format": "2006-01-02 15:04:05 MST", - }, - }) + outputColumns := make([]interface{}, 0) + for _, col := range createTableCmd.Columns { + columnType := strings.TrimSpace(col.ColumnType) + + // Normalize types + + if strings.Contains(columnType, "Nullable") { + columnType = unwrapNullable(columnType) + } + if strings.Contains(columnType, "Map") { + columnType = "map" + } + if strings.Contains(columnType, "DateTime") { + columnType = "datetime" + } + if strings.Contains(columnType, "Array") { + columnType = "array" + } + if strings.Contains(columnType, "Float64") { + columnType = "double" + } + + if columnType == "" { + fmt.Printf("Warning: column %s has empty or unknown type\n", col.ColumnName) + continue // skip malformed types + } + + // Build base datatype map + datatype := map[string]interface{}{ + "type": columnType, + } + + // Optionally add format for datetime + if columnType == "datetime" { + datatype["format"] = "2006-01-02 15:04:05 MST" + } + + if col.ColumnName == "@timestamp" { + datatype["primary"] = true + } + + if columnType == "array" { + datatype["elements"] = []interface{}{ + map[string]interface{}{ + "type": "double", + "index_options": map[string]interface{}{ + "fulltext": false, + }, + }, + } + } + if columnType == "map" { + datatype["elements"] = []interface{}{ + map[string]interface{}{ + "type": "string", + "index_options": map[string]interface{}{ + "fulltext": false, + }, + }, + map[string]interface{}{ + "type": "string", + "index_options": map[string]interface{}{ + "fulltext": false, + }, + }, + } + } columnMap := map[string]interface{}{ - "name": col.ColumnName, - "datatype": map[string]interface{}{ - "type": "uint64", - }, + "name": col.ColumnName, + "datatype": datatype, } outputColumns = append(outputColumns, columnMap) From 9c2076aa91d6a52f850cf11289e7a79533d581a7 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 18:05:35 +0200 Subject: [PATCH 20/54] Handle arrays and maps #1 --- platform/ingest/hydrolixlowerer.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index ba0bcacbb..423af5502 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -180,22 +180,13 @@ func (l *HydrolixLowerer) LowerToDDL( outputColumns := make([]interface{}, 0) for _, col := range createTableCmd.Columns { - columnType := strings.TrimSpace(col.ColumnType) + columnType := strings.TrimSpace(strings.ToLower(col.ColumnType)) // Normalize types if strings.Contains(columnType, "Nullable") { columnType = unwrapNullable(columnType) } - if strings.Contains(columnType, "Map") { - columnType = "map" - } - if strings.Contains(columnType, "DateTime") { - columnType = "datetime" - } - if strings.Contains(columnType, "Array") { - columnType = "array" - } if strings.Contains(columnType, "Float64") { columnType = "double" } From 91266a57a041573ffcd2f9b16a5d8615070ed437 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 18:22:57 +0200 Subject: [PATCH 21/54] Some fixes --- platform/ingest/hydrolixlowerer.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 423af5502..b8b1167c0 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -180,27 +180,37 @@ func (l *HydrolixLowerer) LowerToDDL( outputColumns := make([]interface{}, 0) for _, col := range createTableCmd.Columns { - columnType := strings.TrimSpace(strings.ToLower(col.ColumnType)) + columnType := strings.TrimSpace(col.ColumnType) // Normalize types + var isNullable bool if strings.Contains(columnType, "Nullable") { + isNullable = true columnType = unwrapNullable(columnType) } + if strings.Contains(columnType, "Map") { + columnType = "map" + } + if strings.Contains(columnType, "DateTime") { + columnType = "datetime" + } + if strings.Contains(columnType, "Array") { + columnType = "array" + } if strings.Contains(columnType, "Float64") { columnType = "double" } - if columnType == "" { - fmt.Printf("Warning: column %s has empty or unknown type\n", col.ColumnName) - continue // skip malformed types - } - // Build base datatype map datatype := map[string]interface{}{ "type": columnType, } + if isNullable { + datatype["denullify"] = false + } + // Optionally add format for datetime if columnType == "datetime" { datatype["format"] = "2006-01-02 15:04:05 MST" From e2f9aa8d77a562f1d92ef1b1f2a3d2c30aa79c56 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 18:40:45 +0200 Subject: [PATCH 22/54] Some tooling --- platform/ingest/hydrolixlowerer.go | 121 ++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 4 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index b8b1167c0..2d5bd4f15 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -9,7 +9,6 @@ import ( "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" "github.com/goccy/go-json" - "regexp" "strings" "sync/atomic" ) @@ -73,9 +72,123 @@ func (ip *HydrolixLowerer) GenerateIngestContent(table *chLib.Table, return alterStatements, onlySchemaFields, nonSchemaFields, nil } -func unwrapNullable(input string) string { - re := regexp.MustCompile(`Nullable\(([^)]+)\)`) - return re.ReplaceAllString(input, `$1`) +type TypeId int + +const ( + PrimitiveType TypeId = iota + ArrayType + MapType +) + +type TypeElement struct { + Name string + IsNullable bool +} + +type TypeInfo struct { + TypeId TypeId + Elements []TypeElement + IsNullable bool +} + +func GetTypeInfo(typeName string) TypeInfo { + columnType := strings.TrimSpace(typeName) + info := TypeInfo{} + + // Check for Nullable wrapper + if strings.HasPrefix(columnType, "Nullable(") { + info.IsNullable = true + columnType = unwrapNullable(columnType) + } + + // Parse Array or Map + switch { + case strings.HasPrefix(columnType, "Array("): + info.TypeId = ArrayType + inner := unwrapGeneric(columnType) + info.Elements = []TypeElement{{Name: normalizeType(inner)}} + + case strings.HasPrefix(columnType, "Map("): + info.TypeId = MapType + inner := unwrapGeneric(columnType) + parts := splitCommaArgs(inner) + if len(parts) == 2 { + info.Elements = []TypeElement{ + {Name: normalizeType(parts[0])}, + {Name: normalizeType(parts[1])}, + } + } + + default: + info.TypeId = PrimitiveType + info.Elements = []TypeElement{{Name: normalizeType(columnType)}} + } + + return info +} + +// Unwraps e.g. Array(Float64) → Float64 +func unwrapGeneric(s string) string { + start := strings.Index(s, "(") + end := strings.LastIndex(s, ")") + if start >= 0 && end > start { + return strings.TrimSpace(s[start+1 : end]) + } + return s +} + +// Splits arguments like Map(String, Int64) +func splitCommaArgs(s string) []string { + var args []string + var current strings.Builder + var depth int + for _, r := range s { + switch r { + case '(': + depth++ + current.WriteRune(r) + case ')': + depth-- + current.WriteRune(r) + case ',': + if depth == 0 { + args = append(args, strings.TrimSpace(current.String())) + current.Reset() + } else { + current.WriteRune(r) + } + default: + current.WriteRune(r) + } + } + if trimmed := strings.TrimSpace(current.String()); trimmed != "" { + args = append(args, trimmed) + } + return args +} + +// Normalize ClickHouse-like types +func normalizeType(t string) string { + t = strings.ToLower(strings.TrimSpace(t)) + switch t { + case "float64": + return "double" + case "datetime64", "datetime": + return "datetime" + case "int64": + return "int64" + case "string": + return "string" + } + return t +} + +// Removes Nullable(...) and returns the inner string +func unwrapNullable(s string) string { + if strings.HasPrefix(s, "Nullable(") && strings.HasSuffix(s, ")") { + return strings.TrimSpace(s[9 : len(s)-1]) + } + return s } func (l *HydrolixLowerer) LowerToDDL( From 7b47b3ad7c7e4194a8a378bd302d56608639ce38 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 19:09:12 +0200 Subject: [PATCH 23/54] Refactoring --- platform/ingest/hydrolixlowerer.go | 189 +++++++++-------------------- 1 file changed, 54 insertions(+), 135 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 2d5bd4f15..b764b95cc 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -170,15 +170,11 @@ func splitCommaArgs(s string) []string { // Normalize ClickHouse-like types func normalizeType(t string) string { t = strings.ToLower(strings.TrimSpace(t)) - switch t { - case "float64": + switch { + case strings.Contains(t, "float64"): return "double" - case "datetime64", "datetime": + case strings.Contains(t, "datetime"): return "datetime" - case "int64": - return "int64" - case "string": - return "string" } return t } @@ -198,87 +194,6 @@ func (l *HydrolixLowerer) LowerToDDL( encodings map[schema.FieldEncodingKey]schema.EncodedFieldName, createTableCmd CreateTableStatement, ) ([]string, error) { - /* - // Construct columns array - var columnsJSON strings.Builder - columnsJSON.WriteString("[\n") - - for i, col := range createTableCmd.Columns { - if i > 0 { - columnsJSON.WriteString(",\n") - } - columnsJSON.WriteString(fmt.Sprintf(` { "name": "%s", "type": "%s"`, col.ColumnName, col.ColumnType)) - if col.Comment != "" { - columnsJSON.WriteString(fmt.Sprintf(`, "comment": "%s"`, col.Comment)) - } - if col.AdditionalMetadata != "" { - columnsJSON.WriteString(fmt.Sprintf(`, "metadata": "%s"`, col.AdditionalMetadata)) - } - columnsJSON.WriteString(" }") - } - - columnsJSON.WriteString("\n]") - - const timeColumnName = "ingest_time" - - const ( - partitioningStrategy = "strategy" - partitioningField = "field" - partitioningGranularity = "granularity" - - defaultStrategy = "time" - defaultField = "ingest_time" - defaultGranularity = "day" - ) - partitioningJSON := fmt.Sprintf(`"partitioning": { - "%s": "%s", - "%s": "%s", - "%s": "%s" - }`, - partitioningStrategy, defaultStrategy, - partitioningField, defaultField, - partitioningGranularity, defaultGranularity) - events := make(map[string]any) - for i, preprocessedJson := range validatedJsons { - _, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, - invalidJsons[i], encodings) - if err != nil { - return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) - } - if err != nil { - return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) - } - content := convertNonSchemaFieldsToMap(nonSchemaFields) - - for k, v := range onlySchemaFields { - content[k] = v - } - - for k, v := range content { - events[k] = v - } - } - - eventList := []map[string]any{events} - eventBytes, err := json.MarshalIndent(eventList, " ", " ") - if err != nil { - return nil, err - } - eventJSON := string(eventBytes) - - result := fmt.Sprintf(`{ - "schema": { - "project": "%s", - "name": "%s", - "time_column": "%s", - "columns": %s, - %s, - }, - "events": %s - }`, table.DatabaseName, table.Name, timeColumnName, columnsJSON.String(), partitioningJSON, eventJSON) - return []string{result}, nil - */ - // --- Create Table Section --- createTable := map[string]interface{}{ "name": table.Name, @@ -293,72 +208,76 @@ func (l *HydrolixLowerer) LowerToDDL( outputColumns := make([]interface{}, 0) for _, col := range createTableCmd.Columns { - columnType := strings.TrimSpace(col.ColumnType) - - // Normalize types - - var isNullable bool - if strings.Contains(columnType, "Nullable") { - isNullable = true - columnType = unwrapNullable(columnType) - } - if strings.Contains(columnType, "Map") { - columnType = "map" - } - if strings.Contains(columnType, "DateTime") { - columnType = "datetime" - } - if strings.Contains(columnType, "Array") { - columnType = "array" - } - if strings.Contains(columnType, "Float64") { - columnType = "double" - } + typeInfo := GetTypeInfo(col.ColumnType) // Build base datatype map datatype := map[string]interface{}{ - "type": columnType, + "type": typeInfo.Elements[0].Name, // For primitive, or outer type for array/map } - if isNullable { + // Nullable handling + if typeInfo.IsNullable { datatype["denullify"] = false } - // Optionally add format for datetime - if columnType == "datetime" { - datatype["format"] = "2006-01-02 15:04:05 MST" - } - + // Primary timestamp column if col.ColumnName == "@timestamp" { datatype["primary"] = true } - if columnType == "array" { - datatype["elements"] = []interface{}{ - map[string]interface{}{ - "type": "double", - "index_options": map[string]interface{}{ - "fulltext": false, - }, + // Add format for datetime + if datatype["type"] == "datetime" { + datatype["format"] = "2006-01-02 15:04:05 MST" + } + + // Handle array elements + if typeInfo.TypeId == ArrayType && len(typeInfo.Elements) > 0 { + datatype["type"] = "array" + elementType := normalizeType(typeInfo.Elements[0].Name) + element := map[string]interface{}{ + "type": elementType, + "index_options": map[string]interface{}{ + "fulltext": false, }, } + + if elementType == "datetime" { + element["format"] = "2006-01-02 15:04:05 MST" + } + + datatype["elements"] = []interface{}{element} } - if columnType == "map" { - datatype["elements"] = []interface{}{ - map[string]interface{}{ - "type": "string", - "index_options": map[string]interface{}{ - "fulltext": false, - }, + + // Handle map elements + if typeInfo.TypeId == MapType && len(typeInfo.Elements) == 2 { + datatype["type"] = "map" + keyType := normalizeType(typeInfo.Elements[0].Name) + valueType := normalizeType(typeInfo.Elements[1].Name) + + element1 := map[string]interface{}{ + "type": keyType, + "index_options": map[string]interface{}{ + "fulltext": false, }, - map[string]interface{}{ - "type": "string", - "index_options": map[string]interface{}{ - "fulltext": false, - }, + } + + if keyType == "datetime" { + element1["format"] = "2006-01-02 15:04:05 MST" + } + element2 := map[string]interface{}{ + "type": valueType, + "index_options": map[string]interface{}{ + "fulltext": false, }, } + + if valueType == "datetime" { + element2["format"] = "2006-01-02 15:04:05 MST" + } + datatype["elements"] = []interface{}{element1, element2} } + + // Final column map columnMap := map[string]interface{}{ "name": col.ColumnName, "datatype": datatype, From 8eb48a2902a16d99c90a84a91181c2b91eb29646 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 19:12:58 +0200 Subject: [PATCH 24/54] Clear ids --- platform/backend_connectors/hydrolix_backend_connector.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 1214eab80..4e08d109d 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -106,10 +106,10 @@ var tableCache = make(map[string]uuid.UUID) var tableMutex sync.Mutex func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { - token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI3NDUyMjYsImlhdCI6MTc1MjY1ODgyNiwianRpIjoiMjE1NGQ2YTItNTBiOC00MWM2LTk4MDYtYzE5N2M4YTUzZjJlIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiMzA0NTg5MTItMjkzYS00MDUwLWEwZDMtNmExMzc0ZjkzZTBhIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiMzA0NTg5MTItMjkzYS00MDUwLWEwZDMtNmExMzc0ZjkzZTBhIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.OAqqHSA2UePYIryGVvF4AFhxxTnAbdLa-xnmAWJUoECkZnGlLWdkRYemIq2aoTc2KgWJwa_afh1sXMpXafpfQm_dFNDVk7dAgQMTOfSM8RyrcSJBJTTQDNFriPAmRsfQW4al33tKPj4cr527x00oDs8U8dRQYX0t2c2ae74YTZ1ncE8AQrA48dpBNf9wHjjNxX1m1TQk_129Wvc0BG0QtX3yiir6FtktbBGhDQFSFS-OYdGItFDrfKsMaB7yN3X4GrNU9yv4ZgYUEEHWpU6SL8ATzDX9hWBpotn1svY9BT99N4bvC5iRVeAoOs4dlmv7Ctydgaw8NSSy68SVO6ztfg" - hdxHost := "3.20.203.177:8888" - orgID := "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" - projectID := "27506b30-0c78-41fa-a059-048d687f1164" + token := "" + hdxHost := "" + orgID := "" + projectID := "" if !isValidJSON(query) { return fmt.Errorf("invalid JSON payload: %s", query) From 767863296f903e82a1967f36778b10a47388b61e Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 19:16:47 +0200 Subject: [PATCH 25/54] Fixing linter --- .../backend_connectors/hydrolix_backend_connector.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 4e08d109d..8f5dd117d 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -65,7 +65,7 @@ func isValidJSON(s string) bool { return json.Unmarshal([]byte(s), &js) == nil } -func makeRequest(ctx context.Context, method string, url string, body []byte, token string, tableName string) (error, []byte) { +func makeRequest(ctx context.Context, method string, url string, body []byte, token string, tableName string) ([]byte, error) { // Build the request req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body)) if err != nil { @@ -95,9 +95,9 @@ func makeRequest(ctx context.Context, method string, url string, body []byte, to // Read and print response respBody, err := io.ReadAll(resp.Body) if resp.StatusCode >= 400 { - return fmt.Errorf("ingest failed: %s — %s", resp.Status, string(respBody)), nil + return nil, fmt.Errorf("ingest failed: %s — %s", resp.Status, string(respBody)) } - return err, respBody + return respBody, err } var tableId uuid.UUID @@ -154,7 +154,7 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args if err != nil { return fmt.Errorf("error marshalling create_table JSON: %v", err) } - err, _ = makeRequest(ctx, "POST", url, createTableJson, token, tableName) + _, err = makeRequest(ctx, "POST", url, createTableJson, token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err @@ -166,7 +166,7 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args return fmt.Errorf("error marshalling transform JSON: %v", err) } - err, _ = makeRequest(ctx, "POST", url, transformJson, token, tableName) + _, err = makeRequest(ctx, "POST", url, transformJson, token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err @@ -179,7 +179,7 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args return fmt.Errorf("error marshalling ingest JSON: %v", err) } url := fmt.Sprintf("http://%s/ingest/event", hdxHost) - err, _ = makeRequest(ctx, "POST", url, ingestJson, token, tableName) + _, err = makeRequest(ctx, "POST", url, ingestJson, token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err From 4f66cca82b417227a5f7ee4829962762c0d14d16 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 19:20:11 +0200 Subject: [PATCH 26/54] Skipping test for now --- platform/ingest/insert_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/platform/ingest/insert_test.go b/platform/ingest/insert_test.go index b87a9c08e..a1d3ab6d7 100644 --- a/platform/ingest/insert_test.go +++ b/platform/ingest/insert_test.go @@ -430,6 +430,7 @@ func TestCreateTableIfSomeFieldsExistsInSchemaAlready(t *testing.T) { } func TestHydrolixIngest(t *testing.T) { + t.Skip("TODO: this test is not implemented yet, need to implement the Hydrolix backend connector Exec method") indexName := "test_index" quesmaConfig := &config.QuesmaConfiguration{ From 3abd7cebab19d31291841618084e3345aba3a909 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 16 Jul 2025 22:37:47 +0200 Subject: [PATCH 27/54] Adding hydrolix instance --- platform/database_common/schema.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platform/database_common/schema.go b/platform/database_common/schema.go index 400a41872..343ef01fc 100644 --- a/platform/database_common/schema.go +++ b/platform/database_common/schema.go @@ -169,6 +169,8 @@ func GetInstanceType(instanceName string) InstanceType { return DorisInstance case "doris": return ClickHouseInstance + case "hydrolix": + return ClickHouseInstance default: logger.Fatal().Msgf("unknown instance name: %s", instanceName) return UnknownInstance From 11e40dca3308b34e989b8c586d297b5c1260cb7a Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Thu, 17 Jul 2025 16:27:19 +0200 Subject: [PATCH 28/54] First ingest --- platform/ingest/hydrolixlowerer.go | 53 ++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index b764b95cc..336fb695e 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -187,6 +187,27 @@ func unwrapNullable(s string) string { return s } +func defaultForType(t string) interface{} { + switch t { + case "string": + return "example" + case "int64": + return int64(123) + case "uint64": + return uint64(123) + case "uint32": + return uint32(123) + case "double", "float64": + return "1.23" + case "datetime": + return "2020-02-26 16:01:27 PST" + case "bool": + return true + default: + return nil + } +} + func (l *HydrolixLowerer) LowerToDDL( validatedJsons []types.JSON, table *chLib.Table, @@ -301,13 +322,31 @@ func (l *HydrolixLowerer) LowerToDDL( } // --- Ingest Section --- - ingest := map[string]interface{}{ - "timestamp": "2020-02-26 16:01:27 PST", - "clientId": "29992", - "clientIp": "1.2.3.4/24", - "clientCityCode": 1224, - "resolverIp": "1.4.5.7", - "resolveDuration": "1.234", + ingest := map[string]interface{}{} + + for _, col := range createTableCmd.Columns { + colName := col.ColumnName + typeInfo := GetTypeInfo(col.ColumnType) + + var value interface{} + + switch typeInfo.TypeId { + case PrimitiveType: + value = defaultForType(typeInfo.Elements[0].Name) + + case ArrayType: + elemType := typeInfo.Elements[0].Name + value = []interface{}{defaultForType(elemType)} // array with one sample element + + case MapType: + keyType := typeInfo.Elements[0].Name + valType := typeInfo.Elements[1].Name + value = map[string]interface{}{ + fmt.Sprintf("%v", defaultForType(keyType)): defaultForType(valType), + } + } + + ingest[colName] = value } // --- Final Payload --- From 3bfa1d50523e99fd757ed578c43111f621446d94 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 18 Jul 2025 10:39:07 +0200 Subject: [PATCH 29/54] Ingest more than one row --- platform/ingest/hydrolixlowerer.go | 54 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 336fb695e..22d864029 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -322,38 +322,42 @@ func (l *HydrolixLowerer) LowerToDDL( } // --- Ingest Section --- - ingest := map[string]interface{}{} - - for _, col := range createTableCmd.Columns { - colName := col.ColumnName - typeInfo := GetTypeInfo(col.ColumnType) - - var value interface{} - - switch typeInfo.TypeId { - case PrimitiveType: - value = defaultForType(typeInfo.Elements[0].Name) - - case ArrayType: - elemType := typeInfo.Elements[0].Name - value = []interface{}{defaultForType(elemType)} // array with one sample element - - case MapType: - keyType := typeInfo.Elements[0].Name - valType := typeInfo.Elements[1].Name - value = map[string]interface{}{ - fmt.Sprintf("%v", defaultForType(keyType)): defaultForType(valType), + ingests := make([]map[string]interface{}, 0) + for i := 0; i < 10; i++ { + ingest := map[string]interface{}{} + for _, col := range createTableCmd.Columns { + colName := col.ColumnName + typeInfo := GetTypeInfo(col.ColumnType) + + var value interface{} + + switch typeInfo.TypeId { + case PrimitiveType: + value = defaultForType(typeInfo.Elements[0].Name) + + case ArrayType: + elemType := typeInfo.Elements[0].Name + value = []interface{}{defaultForType(elemType)} // array with one sample element + + case MapType: + keyType := typeInfo.Elements[0].Name + valType := typeInfo.Elements[1].Name + value = map[string]interface{}{ + fmt.Sprintf("%v", defaultForType(keyType)): defaultForType(valType), + } } - } - ingest[colName] = value + ingest[colName] = value + } + if len(ingest) > 0 { + ingests = append(ingests, ingest) + } } - // --- Final Payload --- payload := map[string]interface{}{ "create_table": createTable, "transform": transform, - "ingest": ingest, + "ingest": ingests, } marshaledPayload, err := json.Marshal(payload) From b5ba992965c4d949bc4df8053ccb588962eddea0 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 18 Jul 2025 10:39:38 +0200 Subject: [PATCH 30/54] Some fixes plus introduction goroutine for async exec --- .../hydrolix_backend_connector.go | 75 +++++++++++++------ 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 8f5dd117d..500fac8e7 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -23,10 +23,11 @@ import ( type HydrolixBackendConnector struct { BasicSqlBackendConnector // TODO for now we still have reference for RelationalDbConfiguration for fallback - cfg *config.RelationalDbConfiguration - IngestURL string - AccessToken string - Headers map[string]string + cfg *config.RelationalDbConfiguration + IngestURL string + AccessToken string + Headers map[string]string + createTableChan chan string } func (p *HydrolixBackendConnector) GetId() quesma_api.BackendConnectorType { @@ -43,16 +44,22 @@ func (p *HydrolixBackendConnector) Open() error { } func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration) *HydrolixBackendConnector { + createTableChan := make(chan string) + go listenForCreateTable(createTableChan) return &HydrolixBackendConnector{ - cfg: configuration, + cfg: configuration, + createTableChan: createTableChan, } } func NewHydrolixBackendConnectorWithConnection(_ string, conn *sql.DB) *HydrolixBackendConnector { + createTableChan := make(chan string) + go listenForCreateTable(createTableChan) return &HydrolixBackendConnector{ BasicSqlBackendConnector: BasicSqlBackendConnector{ connection: conn, }, + createTableChan: createTableChan, } } @@ -96,20 +103,26 @@ func makeRequest(ctx context.Context, method string, url string, body []byte, to respBody, err := io.ReadAll(resp.Body) if resp.StatusCode >= 400 { return nil, fmt.Errorf("ingest failed: %s — %s", resp.Status, string(respBody)) + } else { + logger.InfoWithCtx(ctx).Msgf("Ingest successful: %s %s — %s", tableName, resp.Status, string(respBody)) } return respBody, err } -var tableId uuid.UUID -var tableName string var tableCache = make(map[string]uuid.UUID) var tableMutex sync.Mutex +func listenForCreateTable(ch <-chan string) { + for url := range ch { + _ = url // TODO: handle the URL if needed + } +} + func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { - token := "" - hdxHost := "" - orgID := "" - projectID := "" + token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI4MzE4ODUsImlhdCI6MTc1Mjc0NTQ4NSwianRpIjoiMjc2NDNlMmYtYTEyMS00OTE4LWE3MjAtYzM3ZjZkNTA5NjQzIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiMGEyMzNhZGItNjg4ZC00NDY3LWJkMmItMjQxOWRlZDk2MzNjIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiMGEyMzNhZGItNjg4ZC00NDY3LWJkMmItMjQxOWRlZDk2MzNjIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.Zcer0PxmVB9cFe9_p4ubnBcIn4TjDBEXZoezjDoh9CduhyWVPoR3hRghO3JzkqpMavZxrijHlCsFbR31JKk3YZnuJ66Ve_7YwL8FJyzwVg17biW7ESAqRv0cYpmoUIh5AsQT8sagLhdrvX4wmndJvsrGJiGsYn6-YFj-R4Q7qyK4HAGk2IfyRlTeqWSN6FC1y_jgr4IqXB5gU6Y5pnTs782yx-0qd8rMGb6a3h4OFeSz2qS-y0zcDRV8pxyE27RRiN1-cQIL90QtMUMrAcy_qp-YnY15kr_xGbjMpbvDvL-R6xaxHH1DBIfs-QAdgj0IgMe-EBO9-w3h7Vpxmyda1w" + hdxHost := "3.20.203.177:8888" + orgID := "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" + projectID := "27506b30-0c78-41fa-a059-048d687f1164" if !isValidJSON(query) { return fmt.Errorf("invalid JSON payload: %s", query) @@ -124,7 +137,7 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args // Extract each section into its own map (or struct, if needed) var createTable map[string]interface{} var transform map[string]interface{} - var ingest map[string]interface{} + var ingest []map[string]interface{} if err := json.Unmarshal(root["create_table"], &createTable); err != nil { panic(err) @@ -135,7 +148,7 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args if err := json.Unmarshal(root["ingest"], &ingest); err != nil { panic(err) } - + var tableId uuid.UUID // Check if tableId is already cached tableMutex.Lock() if id, exists := tableCache[createTable["name"].(string)]; exists { @@ -147,10 +160,12 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args if len(createTable) > 0 && tableId == uuid.Nil { url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) - tableName = createTable["name"].(string) + tableName := createTable["name"].(string) tableId = uuid.New() createTable["uuid"] = tableId.String() createTableJson, err := json.Marshal(createTable) + logger.Info().Msgf("createtable event: %s %s", createTable["name"].(string), string(createTableJson)) + if err != nil { return fmt.Errorf("error marshalling create_table JSON: %v", err) } @@ -165,24 +180,36 @@ func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args if err != nil { return fmt.Errorf("error marshalling transform JSON: %v", err) } + logger.Info().Msgf("transform event: %s %s", createTable["name"].(string), string(transformJson)) _, err = makeRequest(ctx, "POST", url, transformJson, token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err } + tableMutex.Lock() + tableCache[createTable["name"].(string)] = tableId + tableMutex.Unlock() time.Sleep(5 * time.Second) // Wait for the transform to be created } - if len(ingest) > 0 && tableId != uuid.Nil { - ingestJson, err := json.Marshal(ingest) - if err != nil { - return fmt.Errorf("error marshalling ingest JSON: %v", err) - } - url := fmt.Sprintf("http://%s/ingest/event", hdxHost) - _, err = makeRequest(ctx, "POST", url, ingestJson, token, tableName) - if err != nil { - logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) - return err + + if len(ingest) > 0 { + logger.Info().Msgf("ingests len: %s %d", createTable["name"].(string), len(ingest)) + for _, row := range ingest { + if len(row) == 0 { + continue + } + ingestJson, err := json.Marshal(row) + if err != nil { + return fmt.Errorf("error marshalling ingest JSON: %v", err) + } + url := fmt.Sprintf("http://%s/ingest/event", hdxHost) + logger.Info().Msgf("ingest event: %s %s", createTable["name"].(string), string(ingestJson)) + _, err = makeRequest(ctx, "POST", url, ingestJson, token, createTable["name"].(string)) + if err != nil { + logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) + return err + } } } From 08b3a14b5c39d98e9f2d0153aafe3f528af558ef Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 18 Jul 2025 15:30:42 +0200 Subject: [PATCH 31/54] First real primitive types ingest --- .../hydrolix_backend_connector.go | 5 +- platform/ingest/hydrolixlowerer.go | 72 ++++++++++++++++++- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 500fac8e7..5a5f64a90 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -95,7 +95,7 @@ func makeRequest(ctx context.Context, method string, url string, body []byte, to // Send the request resp, err := client.Do(req) if err != nil { - panic(err) + return nil, fmt.Errorf("ingest request failed: %s", err) } defer resp.Body.Close() @@ -119,7 +119,8 @@ func listenForCreateTable(ch <-chan string) { } func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { - token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI4MzE4ODUsImlhdCI6MTc1Mjc0NTQ4NSwianRpIjoiMjc2NDNlMmYtYTEyMS00OTE4LWE3MjAtYzM3ZjZkNTA5NjQzIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiMGEyMzNhZGItNjg4ZC00NDY3LWJkMmItMjQxOWRlZDk2MzNjIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiMGEyMzNhZGItNjg4ZC00NDY3LWJkMmItMjQxOWRlZDk2MzNjIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.Zcer0PxmVB9cFe9_p4ubnBcIn4TjDBEXZoezjDoh9CduhyWVPoR3hRghO3JzkqpMavZxrijHlCsFbR31JKk3YZnuJ66Ve_7YwL8FJyzwVg17biW7ESAqRv0cYpmoUIh5AsQT8sagLhdrvX4wmndJvsrGJiGsYn6-YFj-R4Q7qyK4HAGk2IfyRlTeqWSN6FC1y_jgr4IqXB5gU6Y5pnTs782yx-0qd8rMGb6a3h4OFeSz2qS-y0zcDRV8pxyE27RRiN1-cQIL90QtMUMrAcy_qp-YnY15kr_xGbjMpbvDvL-R6xaxHH1DBIfs-QAdgj0IgMe-EBO9-w3h7Vpxmyda1w" + // TODO hardcoded for now + token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI5MjUxMzgsImlhdCI6MTc1MjgzODczOCwianRpIjoiNGE0NTQ5ODQtZGJiYS00M2YyLWFlODEtY2YyYmU0ZDRhYWFkIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiOWU0MmI3ZjMtMTc1NS00YTRjLTk1YTMtNDAzZjdkMjdmMmRhIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiOWU0MmI3ZjMtMTc1NS00YTRjLTk1YTMtNDAzZjdkMjdmMmRhIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.fEsqAlbQ0QJiUcxD0MXRd4tei3r-pHfBJHriJuipDE1QsGnZz6jxqrEqRXieM4LlUFxV9ZuGzAZ2QNhorxMbYm5qSGCi7bYZHCDjb0KUWSA2bebNNySppwgObM6U5eShBG8XA5IGFsIj1fXpWmhR80S5LCkwif1t9V6ahV6u15C-f4ZV7DrhpYiEeewKBI0bGme-evTdD0sVN9FpeJszzC5yemP-sFP6v-zammnGbswmRshUGD809r3Sn7kYd5eCir9a76yQ3Es-sanxa6gIpwkpkaPWO6bXUiaA9p9tuD7lEu5iYQ5Q1uln5MzlzMw6Ag1ehDZNCq_zYwmyX9I-Wg" hdxHost := "3.20.203.177:8888" orgID := "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" projectID := "27506b30-0c78-41fa-a059-048d687f1164" diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 22d864029..0b16d0e9b 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -9,6 +9,7 @@ import ( "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" "github.com/goccy/go-json" + "strconv" "strings" "sync/atomic" ) @@ -208,6 +209,52 @@ func defaultForType(t string) interface{} { } } +func CastToType(value any, typeName string) (any, error) { + switch typeName { + case "string": + if v, ok := value.(string); ok { + return v, nil + } + return fmt.Sprintf("%v", value), nil + + case "int": + if v, ok := value.(int); ok { + return v, nil + } + switch v := value.(type) { + case float64: + return int(v), nil + case string: + return strconv.Atoi(v) + } + + case "float64", "double": + if v, ok := value.(float64); ok { + return v, nil + } + switch v := value.(type) { + case int: + return float64(v), nil + case string: + return strconv.ParseFloat(v, 64) + } + + case "bool": + if v, ok := value.(bool); ok { + return v, nil + } + switch v := value.(type) { + case string: + return strconv.ParseBool(v) + } + + default: + return nil, fmt.Errorf("unsupported target type: %s", typeName) + } + + return nil, fmt.Errorf("cannot convert %T to %s", value, typeName) +} + func (l *HydrolixLowerer) LowerToDDL( validatedJsons []types.JSON, table *chLib.Table, @@ -323,17 +370,38 @@ func (l *HydrolixLowerer) LowerToDDL( // --- Ingest Section --- ingests := make([]map[string]interface{}, 0) - for i := 0; i < 10; i++ { + events := make(map[string]any) + for i, preprocessedJson := range validatedJsons { + _, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, + invalidJsons[i], encodings) + if err != nil { + return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) + } + content := convertNonSchemaFieldsToMap(nonSchemaFields) + + for k, v := range onlySchemaFields { + content[k] = v + } + + for k, v := range content { + events[k] = v + } ingest := map[string]interface{}{} for _, col := range createTableCmd.Columns { colName := col.ColumnName + typeInfo := GetTypeInfo(col.ColumnType) var value interface{} switch typeInfo.TypeId { case PrimitiveType: - value = defaultForType(typeInfo.Elements[0].Name) + if _, exists := events[colName]; !exists { + value = defaultForType(typeInfo.Elements[0].Name) + } else { + val, _ := CastToType(events[colName], typeInfo.Elements[0].Name) + value = val //defaultForType(typeInfo.Elements[0].Name) + } case ArrayType: elemType := typeInfo.Elements[0].Name From 5f22d831a402cd7cf7cc4f63ee04d155a66c0f80 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Fri, 18 Jul 2025 16:24:22 +0200 Subject: [PATCH 32/54] Set ctx to background --- platform/backend_connectors/hydrolix_backend_connector.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 5a5f64a90..d416be976 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -118,13 +118,15 @@ func listenForCreateTable(ch <-chan string) { } } -func (p *HydrolixBackendConnector) Exec(ctx context.Context, query string, args ...interface{}) error { +func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args ...interface{}) error { // TODO hardcoded for now token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI5MjUxMzgsImlhdCI6MTc1MjgzODczOCwianRpIjoiNGE0NTQ5ODQtZGJiYS00M2YyLWFlODEtY2YyYmU0ZDRhYWFkIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiOWU0MmI3ZjMtMTc1NS00YTRjLTk1YTMtNDAzZjdkMjdmMmRhIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiOWU0MmI3ZjMtMTc1NS00YTRjLTk1YTMtNDAzZjdkMjdmMmRhIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.fEsqAlbQ0QJiUcxD0MXRd4tei3r-pHfBJHriJuipDE1QsGnZz6jxqrEqRXieM4LlUFxV9ZuGzAZ2QNhorxMbYm5qSGCi7bYZHCDjb0KUWSA2bebNNySppwgObM6U5eShBG8XA5IGFsIj1fXpWmhR80S5LCkwif1t9V6ahV6u15C-f4ZV7DrhpYiEeewKBI0bGme-evTdD0sVN9FpeJszzC5yemP-sFP6v-zammnGbswmRshUGD809r3Sn7kYd5eCir9a76yQ3Es-sanxa6gIpwkpkaPWO6bXUiaA9p9tuD7lEu5iYQ5Q1uln5MzlzMw6Ag1ehDZNCq_zYwmyX9I-Wg" hdxHost := "3.20.203.177:8888" orgID := "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" projectID := "27506b30-0c78-41fa-a059-048d687f1164" + // TODO context might be cancelled too early + ctx := context.Background() if !isValidJSON(query) { return fmt.Errorf("invalid JSON payload: %s", query) } From cecbf90204456c9708f96a549295cd9f775b1e33 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 21 Jul 2025 13:31:54 +0200 Subject: [PATCH 33/54] Refactoring --- .../hydrolix_backend_connector.go | 89 +++++++++++++------ platform/ingest/hydrolixlowerer.go | 7 +- 2 files changed, 65 insertions(+), 31 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index d416be976..68a0b9b3e 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -109,6 +109,44 @@ func makeRequest(ctx context.Context, method string, url string, body []byte, to return respBody, err } +// TODO hardcoded for now +const token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTMxNzUyNDAsImlhdCI6MTc1MzA4ODg0MCwianRpIjoiNWZkYmIyMzktMDk5ZS00M2VmLWFhMzctY2JiMDUxZDRkNDAyIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiNDdmZDE2YjAtZjhkYi00NWI4LWIwNzUtOGYyNTRhNGUxNTFkIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiNDdmZDE2YjAtZjhkYi00NWI4LWIwNzUtOGYyNTRhNGUxNTFkIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.A-61oZS599AQd5HrECfNjVPXdKF2iVZmRGvmZuPiKh3JctL6F00V5InptaH4xZCHqOrda2gLTz5UL4hSx5Fdvk9yrpba5UFdmGhFpkW1jDaHDvOKK_UMGip3wGkAwp0EwEvfqcGhL0Iw96bTUOOlx6LAsSoKvOuLsL5ljBkDIi8vJadVfc01IuQu1AlwlDHpNn1EJbEFQi8cocDTYuG2U8FilFvhHLA77CFysimHxzq0n48vPpxCwtNpzQ9pbbX3ZPsxPoHFojXxvTf0yNWnUR76W3Zw2HlJrlRbiiRqQ4afkTWoxzIb6VAEImemKKtSLS4Ym3X9wuFp_fVLSokj-Q" +const hdxHost = "3.20.203.177:8888" +const orgID = "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" +const projectID = "27506b30-0c78-41fa-a059-048d687f1164" + +type TableInfo struct { + Name string `json:"name"` + UUID string `json:"uuid"` +} + +func isTableExists(tableName string) (bool, error) { + url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) + rawJSON, err := makeRequest(context.Background(), "GET", url, []byte{}, token, "") + if err != nil { + fmt.Println("Error making request:", err) + return false, err + } + + var tables []TableInfo + + // Unmarshal only into our minimal struct + err = json.Unmarshal(rawJSON, &tables) + if err != nil { + panic(err) + } + + // Output result + for _, p := range tables { + if p.Name == tableName { + fmt.Printf("Table %s exists with UUID %s\n", p.Name, p.UUID) + return true, nil + } + } + fmt.Printf("Table %s does not exist\n", tableName) + return false, nil +} + var tableCache = make(map[string]uuid.UUID) var tableMutex sync.Mutex @@ -118,12 +156,28 @@ func listenForCreateTable(ch <-chan string) { } } +func ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName string) error { + logger.Info().Msgf("ingests len: %s %d", tableName, len(ingest)) + for _, row := range ingest { + if len(row) == 0 { + continue + } + ingestJson, err := json.Marshal(row) + if err != nil { + return fmt.Errorf("error marshalling ingest JSON: %v", err) + } + url := fmt.Sprintf("http://%s/ingest/event", hdxHost) + //logger.Info().Msgf("ingest event: %s %s", createTable["name"].(string), string(ingestJson)) + _, err = makeRequest(ctx, "POST", url, ingestJson, token, tableName) + if err != nil { + logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) + return err + } + } + return nil +} + func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args ...interface{}) error { - // TODO hardcoded for now - token := "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTI5MjUxMzgsImlhdCI6MTc1MjgzODczOCwianRpIjoiNGE0NTQ5ODQtZGJiYS00M2YyLWFlODEtY2YyYmU0ZDRhYWFkIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiOWU0MmI3ZjMtMTc1NS00YTRjLTk1YTMtNDAzZjdkMjdmMmRhIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiOWU0MmI3ZjMtMTc1NS00YTRjLTk1YTMtNDAzZjdkMjdmMmRhIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.fEsqAlbQ0QJiUcxD0MXRd4tei3r-pHfBJHriJuipDE1QsGnZz6jxqrEqRXieM4LlUFxV9ZuGzAZ2QNhorxMbYm5qSGCi7bYZHCDjb0KUWSA2bebNNySppwgObM6U5eShBG8XA5IGFsIj1fXpWmhR80S5LCkwif1t9V6ahV6u15C-f4ZV7DrhpYiEeewKBI0bGme-evTdD0sVN9FpeJszzC5yemP-sFP6v-zammnGbswmRshUGD809r3Sn7kYd5eCir9a76yQ3Es-sanxa6gIpwkpkaPWO6bXUiaA9p9tuD7lEu5iYQ5Q1uln5MzlzMw6Ag1ehDZNCq_zYwmyX9I-Wg" - hdxHost := "3.20.203.177:8888" - orgID := "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" - projectID := "27506b30-0c78-41fa-a059-048d687f1164" // TODO context might be cancelled too early ctx := context.Background() @@ -160,10 +214,9 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. tableId = uuid.Nil } tableMutex.Unlock() - + tableName := createTable["name"].(string) if len(createTable) > 0 && tableId == uuid.Nil { url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) - tableName := createTable["name"].(string) tableId = uuid.New() createTable["uuid"] = tableId.String() createTableJson, err := json.Marshal(createTable) @@ -183,7 +236,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. if err != nil { return fmt.Errorf("error marshalling transform JSON: %v", err) } - logger.Info().Msgf("transform event: %s %s", createTable["name"].(string), string(transformJson)) + logger.Info().Msgf("transform event: %s %s", tableName, string(transformJson)) _, err = makeRequest(ctx, "POST", url, transformJson, token, tableName) if err != nil { @@ -191,29 +244,13 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. return err } tableMutex.Lock() - tableCache[createTable["name"].(string)] = tableId + tableCache[tableName] = tableId tableMutex.Unlock() time.Sleep(5 * time.Second) // Wait for the transform to be created } if len(ingest) > 0 { - logger.Info().Msgf("ingests len: %s %d", createTable["name"].(string), len(ingest)) - for _, row := range ingest { - if len(row) == 0 { - continue - } - ingestJson, err := json.Marshal(row) - if err != nil { - return fmt.Errorf("error marshalling ingest JSON: %v", err) - } - url := fmt.Sprintf("http://%s/ingest/event", hdxHost) - logger.Info().Msgf("ingest event: %s %s", createTable["name"].(string), string(ingestJson)) - _, err = makeRequest(ctx, "POST", url, ingestJson, token, createTable["name"].(string)) - if err != nil { - logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) - return err - } - } + ingestFun(ctx, ingest, tableName) } return nil diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 0b16d0e9b..81fd71ef5 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -377,16 +377,13 @@ func (l *HydrolixLowerer) LowerToDDL( if err != nil { return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) } - content := convertNonSchemaFieldsToMap(nonSchemaFields) + events = convertNonSchemaFieldsToMap(nonSchemaFields) for k, v := range onlySchemaFields { - content[k] = v - } - - for k, v := range content { events[k] = v } ingest := map[string]interface{}{} + for _, col := range createTableCmd.Columns { colName := col.ColumnName From 514be8d0e1f71c7003d07acefe3cd6ab47700140 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 21 Jul 2025 16:12:43 +0200 Subject: [PATCH 34/54] Fixes --- .../hydrolix_backend_connector.go | 46 ++++++++++--------- platform/ingest/hydrolixlowerer.go | 19 ++++++-- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 68a0b9b3e..0e610c22c 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -28,6 +28,7 @@ type HydrolixBackendConnector struct { AccessToken string Headers map[string]string createTableChan chan string + client *http.Client } func (p *HydrolixBackendConnector) GetId() quesma_api.BackendConnectorType { @@ -49,6 +50,11 @@ func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration return &HydrolixBackendConnector{ cfg: configuration, createTableChan: createTableChan, + client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + }, } } @@ -60,6 +66,11 @@ func NewHydrolixBackendConnectorWithConnection(_ string, conn *sql.DB) *Hydrolix connection: conn, }, createTableChan: createTableChan, + client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + }, } } @@ -72,7 +83,7 @@ func isValidJSON(s string) bool { return json.Unmarshal([]byte(s), &js) == nil } -func makeRequest(ctx context.Context, method string, url string, body []byte, token string, tableName string) ([]byte, error) { +func (p *HydrolixBackendConnector) makeRequest(ctx context.Context, method string, url string, body []byte, token string, tableName string) ([]byte, error) { // Build the request req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body)) if err != nil { @@ -85,15 +96,8 @@ func makeRequest(ctx context.Context, method string, url string, body []byte, to req.Header.Set("Content-Type", "application/json") req.Header.Set("x-hdx-table", "sample_project."+tableName) - // Allow self-signed certs (equivalent to curl -k) - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - // Send the request - resp, err := client.Do(req) + resp, err := p.client.Do(req) if err != nil { return nil, fmt.Errorf("ingest request failed: %s", err) } @@ -120,9 +124,9 @@ type TableInfo struct { UUID string `json:"uuid"` } -func isTableExists(tableName string) (bool, error) { +func (p *HydrolixBackendConnector) isTableExists(tableName string) (bool, error) { url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) - rawJSON, err := makeRequest(context.Background(), "GET", url, []byte{}, token, "") + rawJSON, err := p.makeRequest(context.Background(), "GET", url, []byte{}, token, "") if err != nil { fmt.Println("Error making request:", err) return false, err @@ -139,16 +143,17 @@ func isTableExists(tableName string) (bool, error) { // Output result for _, p := range tables { if p.Name == tableName { - fmt.Printf("Table %s exists with UUID %s\n", p.Name, p.UUID) + logger.Info().Msgf("Table %s exists with UUID %s\n", p.Name, p.UUID) return true, nil } } - fmt.Printf("Table %s does not exist\n", tableName) + logger.Error().Msgf("Table %s does not exist\n", tableName) return false, nil } var tableCache = make(map[string]uuid.UUID) var tableMutex sync.Mutex +var ingestCounter = 5 * time.Second func listenForCreateTable(ch <-chan string) { for url := range ch { @@ -156,8 +161,9 @@ func listenForCreateTable(ch <-chan string) { } } -func ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName string) error { - logger.Info().Msgf("ingests len: %s %d", tableName, len(ingest)) +func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName string) error { + time.Sleep(5 * time.Second) // wait for table creation + logger.InfoWithCtx(ctx).Msgf("Ingests len: %s %d", tableName, len(ingest)) for _, row := range ingest { if len(row) == 0 { continue @@ -168,7 +174,7 @@ func ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName s } url := fmt.Sprintf("http://%s/ingest/event", hdxHost) //logger.Info().Msgf("ingest event: %s %s", createTable["name"].(string), string(ingestJson)) - _, err = makeRequest(ctx, "POST", url, ingestJson, token, tableName) + _, err = p.makeRequest(ctx, "POST", url, ingestJson, token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err @@ -178,7 +184,6 @@ func ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName s } func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args ...interface{}) error { - // TODO context might be cancelled too early ctx := context.Background() if !isValidJSON(query) { @@ -225,7 +230,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. if err != nil { return fmt.Errorf("error marshalling create_table JSON: %v", err) } - _, err = makeRequest(ctx, "POST", url, createTableJson, token, tableName) + _, err = p.makeRequest(ctx, "POST", url, createTableJson, token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err @@ -238,7 +243,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. } logger.Info().Msgf("transform event: %s %s", tableName, string(transformJson)) - _, err = makeRequest(ctx, "POST", url, transformJson, token, tableName) + _, err = p.makeRequest(ctx, "POST", url, transformJson, token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err @@ -246,11 +251,10 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. tableMutex.Lock() tableCache[tableName] = tableId tableMutex.Unlock() - time.Sleep(5 * time.Second) // Wait for the transform to be created } if len(ingest) > 0 { - ingestFun(ctx, ingest, tableName) + go p.ingestFun(ctx, ingest, tableName) } return nil diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 81fd71ef5..e0df9d475 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -3,8 +3,10 @@ package ingest import ( + "context" "fmt" chLib "github.com/QuesmaOrg/quesma/platform/database_common" + "github.com/QuesmaOrg/quesma/platform/logger" "github.com/QuesmaOrg/quesma/platform/persistence" "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" @@ -15,13 +17,15 @@ import ( ) type HydrolixLowerer struct { - virtualTableStorage persistence.JSONDatabase - ingestCounter atomic.Int64 + virtualTableStorage persistence.JSONDatabase + ingestCounter atomic.Int64 + tableCreteStatementMapping map[*chLib.Table]CreateTableStatement // cache for table creation statements } func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixLowerer { return &HydrolixLowerer{ - virtualTableStorage: virtualTableStorage, + virtualTableStorage: virtualTableStorage, + tableCreteStatementMapping: make(map[*chLib.Table]CreateTableStatement), } } @@ -262,6 +266,13 @@ func (l *HydrolixLowerer) LowerToDDL( encodings map[schema.FieldEncodingKey]schema.EncodedFieldName, createTableCmd CreateTableStatement, ) ([]string, error) { + + if _, exists := l.tableCreteStatementMapping[table]; !exists { + l.tableCreteStatementMapping[table] = createTableCmd + } else { + createTableCmd = l.tableCreteStatementMapping[table] + } + // --- Create Table Section --- createTable := map[string]interface{}{ "name": table.Name, @@ -424,7 +435,7 @@ func (l *HydrolixLowerer) LowerToDDL( "transform": transform, "ingest": ingests, } - + logger.InfoWithCtx(context.Background()).Msgf("Ingesting %d %d %d events into table %s", len(validatedJsons), len(createTableCmd.Columns), len(ingests), table.Name) marshaledPayload, err := json.Marshal(payload) if err != nil { return nil, fmt.Errorf("error marshalling payload: %v", err) From 1e2b3382eee6c704ea31f7da0b86d9dab7b4bd28 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 21 Jul 2025 16:32:57 +0200 Subject: [PATCH 35/54] Omit datetime for now --- platform/backend_connectors/hydrolix_backend_connector.go | 2 ++ platform/ingest/hydrolixlowerer.go | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 0e610c22c..c151d9967 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -179,6 +179,7 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[s logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err } + } return nil } @@ -254,6 +255,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. } if len(ingest) > 0 { + logger.Info().Msgf("Received %d rows for table %s", len(ingest), tableName) go p.ingestFun(ctx, ingest, tableName) } diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index e0df9d475..118b265cb 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -407,8 +407,12 @@ func (l *HydrolixLowerer) LowerToDDL( if _, exists := events[colName]; !exists { value = defaultForType(typeInfo.Elements[0].Name) } else { - val, _ := CastToType(events[colName], typeInfo.Elements[0].Name) - value = val //defaultForType(typeInfo.Elements[0].Name) + if typeInfo.Elements[0].Name == "datetime" { + value = defaultForType(typeInfo.Elements[0].Name) + } else { + val, _ := CastToType(events[colName], typeInfo.Elements[0].Name) + value = val //defaultForType(typeInfo.Elements[0].Name) + } } case ArrayType: From 2fffa03510ec11e2a088fcf181afd0776b848f82 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 21 Jul 2025 17:21:31 +0200 Subject: [PATCH 36/54] Make ingest request again in the case of no table found error --- .../hydrolix_backend_connector.go | 49 ++++++++++++++++--- platform/ingest/hydrolixlowerer.go | 2 +- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index c151d9967..09b1a1bae 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -16,6 +16,7 @@ import ( "github.com/google/uuid" "io" "net/http" + "strings" "sync" "time" ) @@ -124,6 +125,35 @@ type TableInfo struct { UUID string `json:"uuid"` } +type TransformInfo struct { + Name string `json:"name"` + UUID string `json:"uuid"` +} + +type HydrolixResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func (p *HydrolixBackendConnector) getTransforms(tableId string) (bool, error) { + url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", hdxHost, orgID, projectID, tableId) + rawJSON, err := p.makeRequest(context.Background(), "GET", url, []byte{}, token, "") + if err != nil { + fmt.Println("Error making request:", err) + return false, err + } + + var tables []TransformInfo + + // Unmarshal only into our minimal struct + err = json.Unmarshal(rawJSON, &tables) + + if err != nil { + panic(err) + } + return len(tables) > 0, nil +} + func (p *HydrolixBackendConnector) isTableExists(tableName string) (bool, error) { url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) rawJSON, err := p.makeRequest(context.Background(), "GET", url, []byte{}, token, "") @@ -153,7 +183,6 @@ func (p *HydrolixBackendConnector) isTableExists(tableName string) (bool, error) var tableCache = make(map[string]uuid.UUID) var tableMutex sync.Mutex -var ingestCounter = 5 * time.Second func listenForCreateTable(ch <-chan string) { for url := range ch { @@ -161,8 +190,7 @@ func listenForCreateTable(ch <-chan string) { } } -func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName string) error { - time.Sleep(5 * time.Second) // wait for table creation +func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName string, tableId string) error { logger.InfoWithCtx(ctx).Msgf("Ingests len: %s %d", tableName, len(ingest)) for _, row := range ingest { if len(row) == 0 { @@ -174,12 +202,17 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[s } url := fmt.Sprintf("http://%s/ingest/event", hdxHost) //logger.Info().Msgf("ingest event: %s %s", createTable["name"].(string), string(ingestJson)) - _, err = p.makeRequest(ctx, "POST", url, ingestJson, token, tableName) + emitRequest: + respJson, err := p.makeRequest(ctx, "POST", url, ingestJson, token, tableName) if err != nil { - logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) - return err + var resp HydrolixResponse + if err := json.Unmarshal(respJson, &resp); err != nil { + if strings.Contains(resp.Message, "no table") { + time.Sleep(5 * time.Second) + goto emitRequest + } + } } - } return nil } @@ -256,7 +289,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. if len(ingest) > 0 { logger.Info().Msgf("Received %d rows for table %s", len(ingest), tableName) - go p.ingestFun(ctx, ingest, tableName) + go p.ingestFun(ctx, ingest, tableName, tableId.String()) } return nil diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 118b265cb..0131fdd72 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -195,7 +195,7 @@ func unwrapNullable(s string) string { func defaultForType(t string) interface{} { switch t { case "string": - return "example" + return "" case "int64": return int64(123) case "uint64": From 64bd5aa74817ea42149259b00e214c32b716c229 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 21 Jul 2025 17:29:00 +0200 Subject: [PATCH 37/54] Removing unused code --- .../hydrolix_backend_connector.go | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 09b1a1bae..b0408db57 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -120,67 +120,11 @@ const hdxHost = "3.20.203.177:8888" const orgID = "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" const projectID = "27506b30-0c78-41fa-a059-048d687f1164" -type TableInfo struct { - Name string `json:"name"` - UUID string `json:"uuid"` -} - -type TransformInfo struct { - Name string `json:"name"` - UUID string `json:"uuid"` -} - type HydrolixResponse struct { Code int `json:"code"` Message string `json:"message"` } -func (p *HydrolixBackendConnector) getTransforms(tableId string) (bool, error) { - url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", hdxHost, orgID, projectID, tableId) - rawJSON, err := p.makeRequest(context.Background(), "GET", url, []byte{}, token, "") - if err != nil { - fmt.Println("Error making request:", err) - return false, err - } - - var tables []TransformInfo - - // Unmarshal only into our minimal struct - err = json.Unmarshal(rawJSON, &tables) - - if err != nil { - panic(err) - } - return len(tables) > 0, nil -} - -func (p *HydrolixBackendConnector) isTableExists(tableName string) (bool, error) { - url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) - rawJSON, err := p.makeRequest(context.Background(), "GET", url, []byte{}, token, "") - if err != nil { - fmt.Println("Error making request:", err) - return false, err - } - - var tables []TableInfo - - // Unmarshal only into our minimal struct - err = json.Unmarshal(rawJSON, &tables) - if err != nil { - panic(err) - } - - // Output result - for _, p := range tables { - if p.Name == tableName { - logger.Info().Msgf("Table %s exists with UUID %s\n", p.Name, p.UUID) - return true, nil - } - } - logger.Error().Msgf("Table %s does not exist\n", tableName) - return false, nil -} - var tableCache = make(map[string]uuid.UUID) var tableMutex sync.Mutex From 9ba3a4f3a500e9e99acfc2153213bcd6596957cc Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 21 Jul 2025 17:38:12 +0200 Subject: [PATCH 38/54] Fixing static check --- platform/ingest/hydrolixlowerer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 0131fdd72..52d97476d 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -381,14 +381,14 @@ func (l *HydrolixLowerer) LowerToDDL( // --- Ingest Section --- ingests := make([]map[string]interface{}, 0) - events := make(map[string]any) + for i, preprocessedJson := range validatedJsons { _, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, invalidJsons[i], encodings) if err != nil { return nil, fmt.Errorf("error BuildInsertJson, tablename: '%s' : %v", table.Name, err) } - events = convertNonSchemaFieldsToMap(nonSchemaFields) + events := convertNonSchemaFieldsToMap(nonSchemaFields) for k, v := range onlySchemaFields { events[k] = v From ebc9bdc7fc3fc132dcda8a46b01d756fca57a1f5 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 22 Jul 2025 15:28:36 +0200 Subject: [PATCH 39/54] Array handling --- .../hydrolix_backend_connector.go | 3 ++- platform/ingest/hydrolixlowerer.go | 24 ++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index b0408db57..a233a7190 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -115,7 +115,7 @@ func (p *HydrolixBackendConnector) makeRequest(ctx context.Context, method strin } // TODO hardcoded for now -const token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTMxNzUyNDAsImlhdCI6MTc1MzA4ODg0MCwianRpIjoiNWZkYmIyMzktMDk5ZS00M2VmLWFhMzctY2JiMDUxZDRkNDAyIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiNDdmZDE2YjAtZjhkYi00NWI4LWIwNzUtOGYyNTRhNGUxNTFkIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiNDdmZDE2YjAtZjhkYi00NWI4LWIwNzUtOGYyNTRhNGUxNTFkIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.A-61oZS599AQd5HrECfNjVPXdKF2iVZmRGvmZuPiKh3JctL6F00V5InptaH4xZCHqOrda2gLTz5UL4hSx5Fdvk9yrpba5UFdmGhFpkW1jDaHDvOKK_UMGip3wGkAwp0EwEvfqcGhL0Iw96bTUOOlx6LAsSoKvOuLsL5ljBkDIi8vJadVfc01IuQu1AlwlDHpNn1EJbEFQi8cocDTYuG2U8FilFvhHLA77CFysimHxzq0n48vPpxCwtNpzQ9pbbX3ZPsxPoHFojXxvTf0yNWnUR76W3Zw2HlJrlRbiiRqQ4afkTWoxzIb6VAEImemKKtSLS4Ym3X9wuFp_fVLSokj-Q" +const token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTMyNjE3NjksImlhdCI6MTc1MzE3NTM2OSwianRpIjoiMzY2MTY2OGMtOGVjMi00YzkxLTg4ZTItOWQ1YmU3YTdiNGNkIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiYzFjZmE2ZjktOWRhMS00ZDEzLThhYTItZmE3MjMzNWEzZjQxIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiYzFjZmE2ZjktOWRhMS00ZDEzLThhYTItZmE3MjMzNWEzZjQxIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.bSfYTJjJGE3S3IGG32qLwgBZrSZSAIImd4hyoN2KqjPARPhwYCqiW3luSlPc4YL-OHH0YW3YYPM1RyxYQn4CW0DvK9pwYE3qLu7Zt6uVd8Kb88KIHxrXN6mQrIP9-cuYUwyVNVbbGfEYi1aYWSWDJsX9IsmRyIh-aLVHzNXKF7f8I_ek6BdvVDFanoqXVytosWruH7avqJ2kswP99PHHOPJD_SpBhvO014p0-trLM5rWTR3YG6823EEpCSXZ8T4nPZXynXcC8cxTvg_1qVusG4_uBTnqw4LL2HlnYkYM1NPQShkcJvhN3-aygXdsiGgqlsljIpa6cuFkTpANeDbriw" const hdxHost = "3.20.203.177:8888" const orgID = "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" const projectID = "27506b30-0c78-41fa-a059-048d687f1164" @@ -149,6 +149,7 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[s emitRequest: respJson, err := p.makeRequest(ctx, "POST", url, ingestJson, token, tableName) if err != nil { + logger.DebugWithCtx(ctx).Msgf("Error ingesting table %s: %v", tableName, err) var resp HydrolixResponse if err := json.Unmarshal(respJson, &resp); err != nil { if strings.Contains(resp.Message, "no table") { diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 52d97476d..4d51c9dc2 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -251,7 +251,16 @@ func CastToType(value any, typeName string) (any, error) { case string: return strconv.ParseBool(v) } - + case "int64": + if v, ok := value.(int64); ok { + return v, nil + } + switch v := value.(type) { + case float64: + return int64(v), nil + case string: + return strconv.Atoi(v) + } default: return nil, fmt.Errorf("unsupported target type: %s", typeName) } @@ -417,8 +426,17 @@ func (l *HydrolixLowerer) LowerToDDL( case ArrayType: elemType := typeInfo.Elements[0].Name - value = []interface{}{defaultForType(elemType)} // array with one sample element - + value = []any{} + if events[colName] != nil { + for _, elem := range events[colName].([]any) { + castedElem, err := CastToType(elem, elemType) + if err != nil { + logger.ErrorWithCtx(context.Background()).Msgf("Error casting element %v to type %s: %v", elem, elemType, err) + continue + } + value = append(value.([]interface{}), castedElem) + } + } case MapType: keyType := typeInfo.Elements[0].Name valType := typeInfo.Elements[1].Name From 4369ada7dc8b92418bf0a616c8b3f8c8a5979a82 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 22 Jul 2025 16:37:52 +0200 Subject: [PATCH 40/54] Handling datetime --- platform/ingest/hydrolixlowerer.go | 33 ++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 4d51c9dc2..78c2b081d 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "sync/atomic" + "time" ) type HydrolixLowerer struct { @@ -213,6 +214,18 @@ func defaultForType(t string) interface{} { } } +func parseFlexibleTime(input string) (time.Time, error) { + // First try RFC3339 (with timezone) + t, err := time.Parse(time.RFC3339, input) + if err == nil { + return t, nil + } + + // Fallback: try without timezone and assume UTC + layout := "2006-01-02T15:04:05" + return time.ParseInLocation(layout, input, time.UTC) +} + func CastToType(value any, typeName string) (any, error) { switch typeName { case "string": @@ -261,6 +274,17 @@ func CastToType(value any, typeName string) (any, error) { case string: return strconv.Atoi(v) } + case "datetime": + if v, ok := value.(string); ok { + + parsedTime, err := parseFlexibleTime(v) + if err != nil { + fmt.Println("Error parsing time:", err) + return nil, err + } + return parsedTime.Format("2006-01-02 15:04:05 MST"), nil + + } default: return nil, fmt.Errorf("unsupported target type: %s", typeName) } @@ -416,12 +440,9 @@ func (l *HydrolixLowerer) LowerToDDL( if _, exists := events[colName]; !exists { value = defaultForType(typeInfo.Elements[0].Name) } else { - if typeInfo.Elements[0].Name == "datetime" { - value = defaultForType(typeInfo.Elements[0].Name) - } else { - val, _ := CastToType(events[colName], typeInfo.Elements[0].Name) - value = val //defaultForType(typeInfo.Elements[0].Name) - } + val, _ := CastToType(events[colName], typeInfo.Elements[0].Name) + value = val + } case ArrayType: From d1417dafc97bbb75b32aeb50c6a616db64a77c2f Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 23 Jul 2025 14:04:23 +0200 Subject: [PATCH 41/54] Handling map type --- .../hydrolix_backend_connector.go | 2 +- platform/ingest/hydrolixlowerer.go | 20 +++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index a233a7190..6e3db76c4 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -115,7 +115,7 @@ func (p *HydrolixBackendConnector) makeRequest(ctx context.Context, method strin } // TODO hardcoded for now -const token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTMyNjE3NjksImlhdCI6MTc1MzE3NTM2OSwianRpIjoiMzY2MTY2OGMtOGVjMi00YzkxLTg4ZTItOWQ1YmU3YTdiNGNkIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiYzFjZmE2ZjktOWRhMS00ZDEzLThhYTItZmE3MjMzNWEzZjQxIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiYzFjZmE2ZjktOWRhMS00ZDEzLThhYTItZmE3MjMzNWEzZjQxIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.bSfYTJjJGE3S3IGG32qLwgBZrSZSAIImd4hyoN2KqjPARPhwYCqiW3luSlPc4YL-OHH0YW3YYPM1RyxYQn4CW0DvK9pwYE3qLu7Zt6uVd8Kb88KIHxrXN6mQrIP9-cuYUwyVNVbbGfEYi1aYWSWDJsX9IsmRyIh-aLVHzNXKF7f8I_ek6BdvVDFanoqXVytosWruH7avqJ2kswP99PHHOPJD_SpBhvO014p0-trLM5rWTR3YG6823EEpCSXZ8T4nPZXynXcC8cxTvg_1qVusG4_uBTnqw4LL2HlnYkYM1NPQShkcJvhN3-aygXdsiGgqlsljIpa6cuFkTpANeDbriw" +const token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTMzNTQ5ODQsImlhdCI6MTc1MzI2ODU4NCwianRpIjoiN2IxZWNjZGItYTAwNi00Mzk3LTg4MmYtNjBiM2MyYzdmM2JmIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiOTZjNTFmMDMtMmE5MS00YmUwLTg0MTktM2U2MDA1YWIxYWJmIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiOTZjNTFmMDMtMmE5MS00YmUwLTg0MTktM2U2MDA1YWIxYWJmIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.e1lWfxphcCAN3aGCBUKOA8hl4gcuw9oNI60YRqAs2azGrVOCdIIZ8ri-kGn_6QtDKFdBmlFa7kXhiB7PzVgS5N_8QM5GlXWp-8LgvF7mhpmhP84xHWLhbYsxV3xDqLEcjnwxFN3XbVvzg_AWiECNP2qtqN5yqsSUMPR99JLcPnrZA7pXWXMflVvlenvrjlXvdZt6fmfEgXPoA54OBrS6QUDUNMIk9qdsSJCM-n96k7vo3dDCGyO12EoYQB2-yq7VegSNyaKi1BW1Jl33sSF7GQapU4YJ6ixMN_PUTkL0_ZzRWrPR6ry1qtxGz6phZbbj4LmmduvJlLqjcSrcMTbLdg" const hdxHost = "3.20.203.177:8888" const orgID = "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" const projectID = "27506b30-0c78-41fa-a059-048d687f1164" diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 78c2b081d..379e04199 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -459,10 +459,22 @@ func (l *HydrolixLowerer) LowerToDDL( } } case MapType: - keyType := typeInfo.Elements[0].Name - valType := typeInfo.Elements[1].Name - value = map[string]interface{}{ - fmt.Sprintf("%v", defaultForType(keyType)): defaultForType(valType), + if events[colName] != nil { + rawMap, ok := events[colName].(map[string]any) + if ok { + valType := typeInfo.Elements[1].Name + typedMap := make(map[string]any) + for rawKey, rawVal := range rawMap { + castedVal, err := CastToType(rawVal, valType) + if err != nil { + logger.ErrorWithCtx(context.Background()). + Msgf("Error casting map value %v to type %s: %v", rawVal, valType, err) + continue + } + typedMap[rawKey] = castedVal + } + value = typedMap + } } } From 26a0a29093090e263b80f97222f9e3f7c888758d Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Wed, 23 Jul 2025 14:31:17 +0200 Subject: [PATCH 42/54] Attributes changes --- platform/ingest/hydrolixlowerer.go | 157 ++++++++++++++++++++++++++++- platform/ingest/processor.go | 93 ----------------- platform/ingest/sqllowerer.go | 94 +++++++++++++++++ 3 files changed, 248 insertions(+), 96 deletions(-) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 379e04199..19e08b925 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -5,6 +5,8 @@ package ingest import ( "context" "fmt" + "github.com/QuesmaOrg/quesma/platform/comment_metadata" + "github.com/QuesmaOrg/quesma/platform/common_table" chLib "github.com/QuesmaOrg/quesma/platform/database_common" "github.com/QuesmaOrg/quesma/platform/logger" "github.com/QuesmaOrg/quesma/platform/persistence" @@ -13,6 +15,7 @@ import ( "github.com/goccy/go-json" "strconv" "strings" + "sync" "sync/atomic" "time" ) @@ -20,6 +23,8 @@ import ( type HydrolixLowerer struct { virtualTableStorage persistence.JSONDatabase ingestCounter atomic.Int64 + ingestFieldStatistics IngestFieldStatistics + ingestFieldStatisticsLock sync.Mutex tableCreteStatementMapping map[*chLib.Table]CreateTableStatement // cache for table creation statements } @@ -29,6 +34,152 @@ func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixL tableCreteStatementMapping: make(map[*chLib.Table]CreateTableStatement), } } +func (ip *HydrolixLowerer) shouldAlterColumns(table *chLib.Table, attrsMap map[string][]interface{}) (bool, []int) { + attrKeys := getAttributesByArrayName(chLib.DeprecatedAttributesKeyColumn, attrsMap) + alterColumnIndexes := make([]int, 0) + + // this is special case for common table storage + // we do always add columns for common table storage + if table.Name == common_table.TableName { + if len(table.Cols) > alterColumnUpperLimit { + logger.Warn().Msgf("Common table has more than %d columns (alwaysAddColumnLimit)", alterColumnUpperLimit) + } + } + + if len(table.Cols) < alwaysAddColumnLimit || table.Name == common_table.TableName { + // We promote all non-schema fields to columns + // therefore we need to add all attrKeys indexes to alterColumnIndexes + for i := 0; i < len(attrKeys); i++ { + alterColumnIndexes = append(alterColumnIndexes, i) + } + return true, alterColumnIndexes + } + + if len(table.Cols) > alterColumnUpperLimit { + return false, nil + } + ip.ingestFieldStatisticsLock.Lock() + if ip.ingestFieldStatistics == nil { + ip.ingestFieldStatistics = make(IngestFieldStatistics) + } + ip.ingestFieldStatisticsLock.Unlock() + for i := 0; i < len(attrKeys); i++ { + ip.ingestFieldStatisticsLock.Lock() + ip.ingestFieldStatistics[IngestFieldBucketKey{indexName: table.Name, field: attrKeys[i]}]++ + counter := ip.ingestCounter.Add(1) + fieldCounter := ip.ingestFieldStatistics[IngestFieldBucketKey{indexName: table.Name, field: attrKeys[i]}] + // reset statistics every alwaysAddColumnLimit + // for now alwaysAddColumnLimit is used in two contexts + // for defining column limit and for resetting statistics + if counter >= alwaysAddColumnLimit { + ip.ingestCounter.Store(0) + ip.ingestFieldStatistics = make(IngestFieldStatistics) + } + ip.ingestFieldStatisticsLock.Unlock() + // if field is present more or equal fieldFrequency + // during each alwaysAddColumnLimit iteration + // promote it to column + if fieldCounter >= fieldFrequency { + alterColumnIndexes = append(alterColumnIndexes, i) + } + } + if len(alterColumnIndexes) > 0 { + return true, alterColumnIndexes + } + return false, nil +} + +// This function generates ALTER TABLE commands for adding new columns +// to the table based on the attributesMap and the table name +// AttributesMap contains the attributes that are not part of the schema +// Function has side effects, it modifies the table.Cols map +// and removes the attributes that were promoted to columns +func (ip *HydrolixLowerer) generateNewColumns( + attrsMap map[string][]interface{}, + table *chLib.Table, + alteredAttributesIndexes []int, + encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) []AlterStatement { + var alterStatements []AlterStatement + attrKeys := getAttributesByArrayName(chLib.DeprecatedAttributesKeyColumn, attrsMap) + attrTypes := getAttributesByArrayName(chLib.DeprecatedAttributesValueType, attrsMap) + var deleteIndexes []int + + reverseMap := reverseFieldEncoding(encodings, table.Name) + + // HACK Alert: + // We must avoid altering the table.Cols map and reading at the same time. + // This should be protected by a lock or a copy of the table should be used. + // + newColumns := make(map[string]*chLib.Column) + for k, v := range table.Cols { + newColumns[k] = v + } + + for i := range alteredAttributesIndexes { + + columnType := "" + modifiers := "" + + if attrTypes[i] == chLib.UndefinedType { + continue + } + + // Array and Map are not Nullable + if strings.Contains(attrTypes[i], "Array") || strings.Contains(attrTypes[i], "Map") { + columnType = attrTypes[i] + } else { + modifiers = "Nullable" + columnType = fmt.Sprintf("Nullable(%s)", attrTypes[i]) + } + + propertyName := attrKeys[i] + field, ok := reverseMap[schema.EncodedFieldName(attrKeys[i])] + if ok { + propertyName = field.FieldName + } + + metadata := comment_metadata.NewCommentMetadata() + metadata.Values[comment_metadata.ElasticFieldName] = propertyName + comment := metadata.Marshall() + + alterColumn := AlterStatement{ + Type: AddColumn, + TableName: table.Name, + OnCluster: table.ClusterName, + ColumnName: attrKeys[i], + ColumnType: columnType, + } + newColumns[attrKeys[i]] = &chLib.Column{Name: attrKeys[i], Type: chLib.NewBaseType(attrTypes[i]), Modifiers: modifiers, Comment: comment} + alterStatements = append(alterStatements, alterColumn) + + alterColumnComment := AlterStatement{ + Type: CommentColumn, + TableName: table.Name, + OnCluster: table.ClusterName, + ColumnName: attrKeys[i], + Comment: comment, + } + alterStatements = append(alterStatements, alterColumnComment) + + deleteIndexes = append(deleteIndexes, i) + } + + table.Cols = newColumns + + if table.VirtualTable { + err := storeVirtualTable(table, ip.virtualTableStorage) + if err != nil { + logger.Error().Msgf("error storing virtual table: %v", err) + } + } + + for i := len(deleteIndexes) - 1; i >= 0; i-- { + attrsMap[chLib.DeprecatedAttributesKeyColumn] = append(attrsMap[chLib.DeprecatedAttributesKeyColumn][:deleteIndexes[i]], attrsMap[chLib.DeprecatedAttributesKeyColumn][deleteIndexes[i]+1:]...) + attrsMap[chLib.DeprecatedAttributesValueType] = append(attrsMap[chLib.DeprecatedAttributesValueType][:deleteIndexes[i]], attrsMap[chLib.DeprecatedAttributesValueType][deleteIndexes[i]+1:]...) + attrsMap[chLib.DeprecatedAttributesValueColumn] = append(attrsMap[chLib.DeprecatedAttributesValueColumn][:deleteIndexes[i]], attrsMap[chLib.DeprecatedAttributesValueColumn][deleteIndexes[i]+1:]...) + } + return alterStatements +} func (ip *HydrolixLowerer) GenerateIngestContent(table *chLib.Table, data types.JSON, @@ -58,9 +209,9 @@ func (ip *HydrolixLowerer) GenerateIngestContent(table *chLib.Table, // have columns for them var alterStatements []AlterStatement ip.ingestCounter.Add(1) - //if ok, alteredAttributesIndexes := ip.shouldAlterColumns(table, attrsMap); ok { - // alterStatements = ip.generateNewColumns(attrsMap, table, alteredAttributesIndexes, encodings) - //} + if ok, alteredAttributesIndexes := ip.shouldAlterColumns(table, attrsMap); ok { + alterStatements = ip.generateNewColumns(attrsMap, table, alteredAttributesIndexes, encodings) + } // If there are some invalid fields, we need to add them to the attributes map // to not lose them and be able to store them later by // generating correct update query diff --git a/platform/ingest/processor.go b/platform/ingest/processor.go index a82a87ee5..00728a964 100644 --- a/platform/ingest/processor.go +++ b/platform/ingest/processor.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "github.com/ClickHouse/clickhouse-go/v2" - "github.com/QuesmaOrg/quesma/platform/comment_metadata" "github.com/QuesmaOrg/quesma/platform/common_table" "github.com/QuesmaOrg/quesma/platform/config" "github.com/QuesmaOrg/quesma/platform/database_common" @@ -344,98 +343,6 @@ func getAttributesByArrayName(arrayName string, return attributes } -// This function generates ALTER TABLE commands for adding new columns -// to the table based on the attributesMap and the table name -// AttributesMap contains the attributes that are not part of the schema -// Function has side effects, it modifies the table.Cols map -// and removes the attributes that were promoted to columns -func (ip *SqlLowerer) generateNewColumns( - attrsMap map[string][]interface{}, - table *database_common.Table, - alteredAttributesIndexes []int, - encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) []AlterStatement { - var alterStatements []AlterStatement - attrKeys := getAttributesByArrayName(database_common.DeprecatedAttributesKeyColumn, attrsMap) - attrTypes := getAttributesByArrayName(database_common.DeprecatedAttributesValueType, attrsMap) - var deleteIndexes []int - - reverseMap := reverseFieldEncoding(encodings, table.Name) - - // HACK Alert: - // We must avoid altering the table.Cols map and reading at the same time. - // This should be protected by a lock or a copy of the table should be used. - // - newColumns := make(map[string]*database_common.Column) - for k, v := range table.Cols { - newColumns[k] = v - } - - for i := range alteredAttributesIndexes { - - columnType := "" - modifiers := "" - - if attrTypes[i] == database_common.UndefinedType { - continue - } - - // Array and Map are not Nullable - if strings.Contains(attrTypes[i], "Array") || strings.Contains(attrTypes[i], "Map") { - columnType = attrTypes[i] - } else { - modifiers = "Nullable" - columnType = fmt.Sprintf("Nullable(%s)", attrTypes[i]) - } - - propertyName := attrKeys[i] - field, ok := reverseMap[schema.EncodedFieldName(attrKeys[i])] - if ok { - propertyName = field.FieldName - } - - metadata := comment_metadata.NewCommentMetadata() - metadata.Values[comment_metadata.ElasticFieldName] = propertyName - comment := metadata.Marshall() - - alterColumn := AlterStatement{ - Type: AddColumn, - TableName: table.Name, - OnCluster: table.ClusterName, - ColumnName: attrKeys[i], - ColumnType: columnType, - } - newColumns[attrKeys[i]] = &database_common.Column{Name: attrKeys[i], Type: database_common.NewBaseType(attrTypes[i]), Modifiers: modifiers, Comment: comment} - alterStatements = append(alterStatements, alterColumn) - - alterColumnComment := AlterStatement{ - Type: CommentColumn, - TableName: table.Name, - OnCluster: table.ClusterName, - ColumnName: attrKeys[i], - Comment: comment, - } - alterStatements = append(alterStatements, alterColumnComment) - - deleteIndexes = append(deleteIndexes, i) - } - - table.Cols = newColumns - - if table.VirtualTable { - err := storeVirtualTable(table, ip.virtualTableStorage) - if err != nil { - logger.Error().Msgf("error storing virtual table: %v", err) - } - } - - for i := len(deleteIndexes) - 1; i >= 0; i-- { - attrsMap[database_common.DeprecatedAttributesKeyColumn] = append(attrsMap[database_common.DeprecatedAttributesKeyColumn][:deleteIndexes[i]], attrsMap[database_common.DeprecatedAttributesKeyColumn][deleteIndexes[i]+1:]...) - attrsMap[database_common.DeprecatedAttributesValueType] = append(attrsMap[database_common.DeprecatedAttributesValueType][:deleteIndexes[i]], attrsMap[database_common.DeprecatedAttributesValueType][deleteIndexes[i]+1:]...) - attrsMap[database_common.DeprecatedAttributesValueColumn] = append(attrsMap[database_common.DeprecatedAttributesValueColumn][:deleteIndexes[i]], attrsMap[database_common.DeprecatedAttributesValueColumn][deleteIndexes[i]+1:]...) - } - return alterStatements -} - // This struct contains the information about the columns that aren't part of the schema // and will go into attributes map type NonSchemaField struct { diff --git a/platform/ingest/sqllowerer.go b/platform/ingest/sqllowerer.go index 88b327678..f585a6f6c 100644 --- a/platform/ingest/sqllowerer.go +++ b/platform/ingest/sqllowerer.go @@ -4,7 +4,9 @@ package ingest import ( "fmt" + "github.com/QuesmaOrg/quesma/platform/comment_metadata" chLib "github.com/QuesmaOrg/quesma/platform/database_common" + "github.com/QuesmaOrg/quesma/platform/logger" "github.com/QuesmaOrg/quesma/platform/persistence" "github.com/QuesmaOrg/quesma/platform/schema" "github.com/QuesmaOrg/quesma/platform/types" @@ -27,6 +29,98 @@ func NewSqlLowerer(virtualTableStorage persistence.JSONDatabase) *SqlLowerer { } } +// This function generates ALTER TABLE commands for adding new columns +// to the table based on the attributesMap and the table name +// AttributesMap contains the attributes that are not part of the schema +// Function has side effects, it modifies the table.Cols map +// and removes the attributes that were promoted to columns +func (ip *SqlLowerer) generateNewColumns( + attrsMap map[string][]interface{}, + table *chLib.Table, + alteredAttributesIndexes []int, + encodings map[schema.FieldEncodingKey]schema.EncodedFieldName) []AlterStatement { + var alterStatements []AlterStatement + attrKeys := getAttributesByArrayName(chLib.DeprecatedAttributesKeyColumn, attrsMap) + attrTypes := getAttributesByArrayName(chLib.DeprecatedAttributesValueType, attrsMap) + var deleteIndexes []int + + reverseMap := reverseFieldEncoding(encodings, table.Name) + + // HACK Alert: + // We must avoid altering the table.Cols map and reading at the same time. + // This should be protected by a lock or a copy of the table should be used. + // + newColumns := make(map[string]*chLib.Column) + for k, v := range table.Cols { + newColumns[k] = v + } + + for i := range alteredAttributesIndexes { + + columnType := "" + modifiers := "" + + if attrTypes[i] == chLib.UndefinedType { + continue + } + + // Array and Map are not Nullable + if strings.Contains(attrTypes[i], "Array") || strings.Contains(attrTypes[i], "Map") { + columnType = attrTypes[i] + } else { + modifiers = "Nullable" + columnType = fmt.Sprintf("Nullable(%s)", attrTypes[i]) + } + + propertyName := attrKeys[i] + field, ok := reverseMap[schema.EncodedFieldName(attrKeys[i])] + if ok { + propertyName = field.FieldName + } + + metadata := comment_metadata.NewCommentMetadata() + metadata.Values[comment_metadata.ElasticFieldName] = propertyName + comment := metadata.Marshall() + + alterColumn := AlterStatement{ + Type: AddColumn, + TableName: table.Name, + OnCluster: table.ClusterName, + ColumnName: attrKeys[i], + ColumnType: columnType, + } + newColumns[attrKeys[i]] = &chLib.Column{Name: attrKeys[i], Type: chLib.NewBaseType(attrTypes[i]), Modifiers: modifiers, Comment: comment} + alterStatements = append(alterStatements, alterColumn) + + alterColumnComment := AlterStatement{ + Type: CommentColumn, + TableName: table.Name, + OnCluster: table.ClusterName, + ColumnName: attrKeys[i], + Comment: comment, + } + alterStatements = append(alterStatements, alterColumnComment) + + deleteIndexes = append(deleteIndexes, i) + } + + table.Cols = newColumns + + if table.VirtualTable { + err := storeVirtualTable(table, ip.virtualTableStorage) + if err != nil { + logger.Error().Msgf("error storing virtual table: %v", err) + } + } + + for i := len(deleteIndexes) - 1; i >= 0; i-- { + attrsMap[chLib.DeprecatedAttributesKeyColumn] = append(attrsMap[chLib.DeprecatedAttributesKeyColumn][:deleteIndexes[i]], attrsMap[chLib.DeprecatedAttributesKeyColumn][deleteIndexes[i]+1:]...) + attrsMap[chLib.DeprecatedAttributesValueType] = append(attrsMap[chLib.DeprecatedAttributesValueType][:deleteIndexes[i]], attrsMap[chLib.DeprecatedAttributesValueType][deleteIndexes[i]+1:]...) + attrsMap[chLib.DeprecatedAttributesValueColumn] = append(attrsMap[chLib.DeprecatedAttributesValueColumn][:deleteIndexes[i]], attrsMap[chLib.DeprecatedAttributesValueColumn][deleteIndexes[i]+1:]...) + } + return alterStatements +} + func (ip *SqlLowerer) GenerateIngestContent(table *chLib.Table, data types.JSON, inValidJson types.JSON, From 4d145baf9d958098fabd7c892fd6a8ba902ce1f9 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 28 Jul 2025 13:03:04 +0200 Subject: [PATCH 43/54] Fixing data amount ingestion --- .../hydrolix_backend_connector.go | 67 +++++++++++++++---- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 6e3db76c4..4e10b4c1d 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -9,12 +9,14 @@ import ( "crypto/tls" "database/sql" "encoding/json" + "errors" "fmt" "github.com/QuesmaOrg/quesma/platform/config" "github.com/QuesmaOrg/quesma/platform/logger" quesma_api "github.com/QuesmaOrg/quesma/platform/v2/core" "github.com/google/uuid" "io" + "net" "net/http" "strings" "sync" @@ -53,7 +55,8 @@ func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration createTableChan: createTableChan, client: &http.Client{ Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DisableKeepAlives: true, }, }, } @@ -69,7 +72,8 @@ func NewHydrolixBackendConnectorWithConnection(_ string, conn *sql.DB) *Hydrolix createTableChan: createTableChan, client: &http.Client{ Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DisableKeepAlives: true, }, }, } @@ -108,14 +112,12 @@ func (p *HydrolixBackendConnector) makeRequest(ctx context.Context, method strin respBody, err := io.ReadAll(resp.Body) if resp.StatusCode >= 400 { return nil, fmt.Errorf("ingest failed: %s — %s", resp.Status, string(respBody)) - } else { - logger.InfoWithCtx(ctx).Msgf("Ingest successful: %s %s — %s", tableName, resp.Status, string(respBody)) } return respBody, err } // TODO hardcoded for now -const token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTMzNTQ5ODQsImlhdCI6MTc1MzI2ODU4NCwianRpIjoiN2IxZWNjZGItYTAwNi00Mzk3LTg4MmYtNjBiM2MyYzdmM2JmIiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiOTZjNTFmMDMtMmE5MS00YmUwLTg0MTktM2U2MDA1YWIxYWJmIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiOTZjNTFmMDMtMmE5MS00YmUwLTg0MTktM2U2MDA1YWIxYWJmIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.e1lWfxphcCAN3aGCBUKOA8hl4gcuw9oNI60YRqAs2azGrVOCdIIZ8ri-kGn_6QtDKFdBmlFa7kXhiB7PzVgS5N_8QM5GlXWp-8LgvF7mhpmhP84xHWLhbYsxV3xDqLEcjnwxFN3XbVvzg_AWiECNP2qtqN5yqsSUMPR99JLcPnrZA7pXWXMflVvlenvrjlXvdZt6fmfEgXPoA54OBrS6QUDUNMIk9qdsSJCM-n96k7vo3dDCGyO12EoYQB2-yq7VegSNyaKi1BW1Jl33sSF7GQapU4YJ6ixMN_PUTkL0_ZzRWrPR6ry1qtxGz6phZbbj4LmmduvJlLqjcSrcMTbLdg" +const token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTM3NzY2NTksImlhdCI6MTc1MzY5MDI1OSwianRpIjoiMzNmNzI2M2MtMTA2Zi00MTc1LWJhZTEtOTEzNTJkNTdmOWM0IiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiNGRhZWM2YzItMzA4ZC00MzFkLTg0ZWMtNGFiMjJjOTFmZjg3IiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiNGRhZWM2YzItMzA4ZC00MzFkLTg0ZWMtNGFiMjJjOTFmZjg3IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.Yr0hleV6sJZCmOQKXSN82HVRm4RKC7IGW7CVXHJai8vOKMW5uPIiw_1BwaHzKi8DjwftHvhWW0hmEXh492Mj_6csQgvejeCfwbKvZx9rQbBZ-4P4GboB4OgqtZ5macY6D_QQyeXol2otS80E8OTAUBM8o07v_fYd92-nz-qY7ceicT8oI7kLMgEOD6VA7Glue7hqQblofIZMoDK1Ve2WhrOhfgqVDxCloFrLs1VhXevGBkVgz7LF_XoxLyR0UPhyVj7lM3ep3M8FJbuP5afKuJUr2nb3qm5Bxs_r1uuQe7INuEH-CYCPJmsOArJ0BIULgtB3LW1zCsLl_DAMQJhwtg" const hdxHost = "3.20.203.177:8888" const orgID = "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" const projectID = "27506b30-0c78-41fa-a059-048d687f1164" @@ -134,32 +136,69 @@ func listenForCreateTable(ch <-chan string) { } } +func isConnectionReset(err error) bool { + // Look for specific substrings or types indicating connection reset + var netErr net.Error + if errors.As(err, &netErr) { + // You may add extra checks here + } + // Match known error message + return strings.Contains(err.Error(), "connection reset by peer") +} + func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName string, tableId string) error { logger.InfoWithCtx(ctx).Msgf("Ingests len: %s %d", tableName, len(ingest)) + + var data []json.RawMessage + for _, row := range ingest { if len(row) == 0 { continue } ingestJson, err := json.Marshal(row) if err != nil { - return fmt.Errorf("error marshalling ingest JSON: %v", err) + logger.ErrorWithCtx(ctx).Msg("Failed to marshal row") + continue } - url := fmt.Sprintf("http://%s/ingest/event", hdxHost) - //logger.Info().Msgf("ingest event: %s %s", createTable["name"].(string), string(ingestJson)) - emitRequest: - respJson, err := p.makeRequest(ctx, "POST", url, ingestJson, token, tableName) + data = append(data, ingestJson) + } + + // Final payload: a JSON array of the rows + finalJson, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal final JSON array: %w", err) + } + + url := fmt.Sprintf("http://%s/ingest/event", hdxHost) + for { + respJson, err := p.makeRequest(ctx, "POST", url, finalJson, token, tableName) if err != nil { - logger.DebugWithCtx(ctx).Msgf("Error ingesting table %s: %v", tableName, err) + logger.ErrorWithCtx(ctx).Msgf("Error ingesting table %s: %v", tableName, err) + + // Retry on connection reset + if isConnectionReset(err) { + logger.WarnWithCtx(ctx).Msgf("Connection reset while ingesting table %s, retrying...", tableName) + time.Sleep(5 * time.Second) + continue + } + + // Try to inspect response (even if err is non-nil) var resp HydrolixResponse - if err := json.Unmarshal(respJson, &resp); err != nil { + if len(respJson) > 0 && json.Unmarshal(respJson, &resp) == nil { if strings.Contains(resp.Message, "no table") { + logger.WarnWithCtx(ctx).Msgf("Table %s not found yet, retrying...", tableName) time.Sleep(5 * time.Second) - goto emitRequest + continue } } + + // If it's another kind of error — continue to the next iteration + continue } + + logger.InfoWithCtx(ctx).Msgf("Ingests successfull: %s %d", tableName, len(ingest)) + return nil } - return nil } func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args ...interface{}) error { From df31f656711da18168b1b5f83b7977b4bd8c5400 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 28 Jul 2025 14:47:38 +0200 Subject: [PATCH 44/54] Simplifying --- .../hydrolix_backend_connector.go | 50 ++----------------- 1 file changed, 4 insertions(+), 46 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 4e10b4c1d..2cb31c81e 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -9,16 +9,13 @@ import ( "crypto/tls" "database/sql" "encoding/json" - "errors" "fmt" "github.com/QuesmaOrg/quesma/platform/config" "github.com/QuesmaOrg/quesma/platform/logger" quesma_api "github.com/QuesmaOrg/quesma/platform/v2/core" "github.com/google/uuid" "io" - "net" "net/http" - "strings" "sync" "time" ) @@ -48,11 +45,8 @@ func (p *HydrolixBackendConnector) Open() error { } func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration) *HydrolixBackendConnector { - createTableChan := make(chan string) - go listenForCreateTable(createTableChan) return &HydrolixBackendConnector{ - cfg: configuration, - createTableChan: createTableChan, + cfg: configuration, client: &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -63,13 +57,10 @@ func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration } func NewHydrolixBackendConnectorWithConnection(_ string, conn *sql.DB) *HydrolixBackendConnector { - createTableChan := make(chan string) - go listenForCreateTable(createTableChan) return &HydrolixBackendConnector{ BasicSqlBackendConnector: BasicSqlBackendConnector{ connection: conn, }, - createTableChan: createTableChan, client: &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -130,22 +121,6 @@ type HydrolixResponse struct { var tableCache = make(map[string]uuid.UUID) var tableMutex sync.Mutex -func listenForCreateTable(ch <-chan string) { - for url := range ch { - _ = url // TODO: handle the URL if needed - } -} - -func isConnectionReset(err error) bool { - // Look for specific substrings or types indicating connection reset - var netErr net.Error - if errors.As(err, &netErr) { - // You may add extra checks here - } - // Match known error message - return strings.Contains(err.Error(), "connection reset by peer") -} - func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName string, tableId string) error { logger.InfoWithCtx(ctx).Msgf("Ingests len: %s %d", tableName, len(ingest)) @@ -170,29 +145,12 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[s } url := fmt.Sprintf("http://%s/ingest/event", hdxHost) + const sleepDuration = 5 * time.Second for { - respJson, err := p.makeRequest(ctx, "POST", url, finalJson, token, tableName) + _, err := p.makeRequest(ctx, "POST", url, finalJson, token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("Error ingesting table %s: %v", tableName, err) - - // Retry on connection reset - if isConnectionReset(err) { - logger.WarnWithCtx(ctx).Msgf("Connection reset while ingesting table %s, retrying...", tableName) - time.Sleep(5 * time.Second) - continue - } - - // Try to inspect response (even if err is non-nil) - var resp HydrolixResponse - if len(respJson) > 0 && json.Unmarshal(respJson, &resp) == nil { - if strings.Contains(resp.Message, "no table") { - logger.WarnWithCtx(ctx).Msgf("Table %s not found yet, retrying...", tableName) - time.Sleep(5 * time.Second) - continue - } - } - - // If it's another kind of error — continue to the next iteration + time.Sleep(sleepDuration) continue } From 0ae0addd303801d784fe72cecbf5c6c7c6565059 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 28 Jul 2025 15:04:34 +0200 Subject: [PATCH 45/54] Improvements --- .../hydrolix_backend_connector.go | 35 ++++++++++--------- platform/ingest/hydrolixlowerer.go | 8 ++--- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 2cb31c81e..8c5669848 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -23,12 +23,11 @@ import ( type HydrolixBackendConnector struct { BasicSqlBackendConnector // TODO for now we still have reference for RelationalDbConfiguration for fallback - cfg *config.RelationalDbConfiguration - IngestURL string - AccessToken string - Headers map[string]string - createTableChan chan string - client *http.Client + cfg *config.RelationalDbConfiguration + IngestURL string + AccessToken string + Headers map[string]string + client *http.Client } func (p *HydrolixBackendConnector) GetId() quesma_api.BackendConnectorType { @@ -121,12 +120,12 @@ type HydrolixResponse struct { var tableCache = make(map[string]uuid.UUID) var tableMutex sync.Mutex -func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[string]interface{}, tableName string, tableId string) error { - logger.InfoWithCtx(ctx).Msgf("Ingests len: %s %d", tableName, len(ingest)) +func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingestSlice []map[string]interface{}, tableName string, tableId string) error { + logger.InfoWithCtx(ctx).Msgf("Ingests len: %s %d", tableName, len(ingestSlice)) var data []json.RawMessage - for _, row := range ingest { + for _, row := range ingestSlice { if len(row) == 0 { continue } @@ -146,17 +145,19 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingest []map[s url := fmt.Sprintf("http://%s/ingest/event", hdxHost) const sleepDuration = 5 * time.Second - for { + const maxRetries = 5 + for retries := 0; retries < maxRetries; retries++ { _, err := p.makeRequest(ctx, "POST", url, finalJson, token, tableName) if err != nil { - logger.ErrorWithCtx(ctx).Msgf("Error ingesting table %s: %v", tableName, err) + logger.WarnWithCtx(ctx).Msgf("Error ingesting table %s: %v retrying...", tableName, err) time.Sleep(sleepDuration) continue } - logger.InfoWithCtx(ctx).Msgf("Ingests successfull: %s %d", tableName, len(ingest)) + logger.InfoWithCtx(ctx).Msgf("Ingests successfull: %s %d", tableName, len(ingestSlice)) return nil } + return fmt.Errorf("failed to ingest after %d retries: %s", maxRetries, tableName) } func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args ...interface{}) error { @@ -175,7 +176,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. // Extract each section into its own map (or struct, if needed) var createTable map[string]interface{} var transform map[string]interface{} - var ingest []map[string]interface{} + var ingestSlice []map[string]interface{} if err := json.Unmarshal(root["create_table"], &createTable); err != nil { panic(err) @@ -183,7 +184,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. if err := json.Unmarshal(root["transform"], &transform); err != nil { panic(err) } - if err := json.Unmarshal(root["ingest"], &ingest); err != nil { + if err := json.Unmarshal(root["ingest"], &ingestSlice); err != nil { panic(err) } var tableId uuid.UUID @@ -229,9 +230,9 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. tableMutex.Unlock() } - if len(ingest) > 0 { - logger.Info().Msgf("Received %d rows for table %s", len(ingest), tableName) - go p.ingestFun(ctx, ingest, tableName, tableId.String()) + if len(ingestSlice) > 0 { + logger.Info().Msgf("Received %d rows for table %s", len(ingestSlice), tableName) + go p.ingestFun(ctx, ingestSlice, tableName, tableId.String()) } return nil diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 19e08b925..0552fc8fb 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -564,7 +564,7 @@ func (l *HydrolixLowerer) LowerToDDL( } // --- Ingest Section --- - ingests := make([]map[string]interface{}, 0) + ingestSlice := make([]map[string]interface{}, 0) for i, preprocessedJson := range validatedJsons { _, onlySchemaFields, nonSchemaFields, err := l.GenerateIngestContent(table, preprocessedJson, @@ -632,16 +632,16 @@ func (l *HydrolixLowerer) LowerToDDL( ingest[colName] = value } if len(ingest) > 0 { - ingests = append(ingests, ingest) + ingestSlice = append(ingestSlice, ingest) } } // --- Final Payload --- payload := map[string]interface{}{ "create_table": createTable, "transform": transform, - "ingest": ingests, + "ingest": ingestSlice, } - logger.InfoWithCtx(context.Background()).Msgf("Ingesting %d %d %d events into table %s", len(validatedJsons), len(createTableCmd.Columns), len(ingests), table.Name) + logger.InfoWithCtx(context.Background()).Msgf("Ingesting %d %d %d events into table %s", len(validatedJsons), len(createTableCmd.Columns), len(ingestSlice), table.Name) marshaledPayload, err := json.Marshal(payload) if err != nil { return nil, fmt.Errorf("error marshalling payload: %v", err) From eb53768b87eb527d97d138abe716afc06112866c Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 28 Jul 2025 15:31:43 +0200 Subject: [PATCH 46/54] Improvements #2 --- .../hydrolix_backend_connector.go | 83 +++++++++++-------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 8c5669848..b34c133b7 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -160,6 +160,50 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingestSlice [] return fmt.Errorf("failed to ingest after %d retries: %s", maxRetries, tableName) } +func getTableIdFromCache(tableName string) (uuid.UUID, bool) { + tableMutex.Lock() + defer tableMutex.Unlock() + id, exists := tableCache[tableName] + return id, exists +} + +func setTableIdInCache(tableName string, tableId uuid.UUID) { + tableMutex.Lock() + defer tableMutex.Unlock() + tableCache[tableName] = tableId +} + +func (p *HydrolixBackendConnector) createTableWithSchema(ctx context.Context, + createTable map[string]interface{}, transform map[string]interface{}, + tableName string, tableId uuid.UUID) error { + url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) + createTableJson, err := json.Marshal(createTable) + logger.Info().Msgf("createtable event: %s %s", tableName, string(createTableJson)) + + if err != nil { + return fmt.Errorf("error marshalling create_table JSON: %v", err) + } + _, err = p.makeRequest(ctx, "POST", url, createTableJson, token, tableName) + if err != nil { + logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) + return err + } + + url = fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", hdxHost, orgID, projectID, tableId.String()) + transformJson, err := json.Marshal(transform) + if err != nil { + return fmt.Errorf("error marshalling transform JSON: %v", err) + } + logger.Info().Msgf("transform event: %s %s", tableName, string(transformJson)) + + _, err = p.makeRequest(ctx, "POST", url, transformJson, token, tableName) + if err != nil { + logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) + return err + } + return nil +} + func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args ...interface{}) error { // TODO context might be cancelled too early ctx := context.Background() @@ -187,47 +231,18 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. if err := json.Unmarshal(root["ingest"], &ingestSlice); err != nil { panic(err) } - var tableId uuid.UUID - // Check if tableId is already cached - tableMutex.Lock() - if id, exists := tableCache[createTable["name"].(string)]; exists { - tableId = id - } else { - tableId = uuid.Nil - } - tableMutex.Unlock() tableName := createTable["name"].(string) + + tableId, _ := getTableIdFromCache(tableName) if len(createTable) > 0 && tableId == uuid.Nil { - url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) tableId = uuid.New() createTable["uuid"] = tableId.String() - createTableJson, err := json.Marshal(createTable) - logger.Info().Msgf("createtable event: %s %s", createTable["name"].(string), string(createTableJson)) - - if err != nil { - return fmt.Errorf("error marshalling create_table JSON: %v", err) - } - _, err = p.makeRequest(ctx, "POST", url, createTableJson, token, tableName) - if err != nil { - logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) - return err - } - - url = fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", hdxHost, orgID, projectID, tableId.String()) - transformJson, err := json.Marshal(transform) - if err != nil { - return fmt.Errorf("error marshalling transform JSON: %v", err) - } - logger.Info().Msgf("transform event: %s %s", tableName, string(transformJson)) - - _, err = p.makeRequest(ctx, "POST", url, transformJson, token, tableName) + err := p.createTableWithSchema(ctx, createTable, transform, tableName, tableId) if err != nil { - logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) + logger.ErrorWithCtx(ctx).Msgf("error creating table with schema: %v", err) return err } - tableMutex.Lock() - tableCache[tableName] = tableId - tableMutex.Unlock() + setTableIdInCache(tableName, tableId) } if len(ingestSlice) > 0 { From 66e85951b621378c085f95e9d55699da4d299136 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 28 Jul 2025 15:39:23 +0200 Subject: [PATCH 47/54] Improvements #3 --- .../hydrolix_backend_connector.go | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index b34c133b7..019537a2a 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -28,6 +28,8 @@ type HydrolixBackendConnector struct { AccessToken string Headers map[string]string client *http.Client + tableCache map[string]uuid.UUID + tableMutex sync.Mutex } func (p *HydrolixBackendConnector) GetId() quesma_api.BackendConnectorType { @@ -52,6 +54,7 @@ func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration DisableKeepAlives: true, }, }, + tableCache: make(map[string]uuid.UUID), } } @@ -66,6 +69,7 @@ func NewHydrolixBackendConnectorWithConnection(_ string, conn *sql.DB) *Hydrolix DisableKeepAlives: true, }, }, + tableCache: make(map[string]uuid.UUID), } } @@ -117,9 +121,6 @@ type HydrolixResponse struct { Message string `json:"message"` } -var tableCache = make(map[string]uuid.UUID) -var tableMutex sync.Mutex - func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingestSlice []map[string]interface{}, tableName string, tableId string) error { logger.InfoWithCtx(ctx).Msgf("Ingests len: %s %d", tableName, len(ingestSlice)) @@ -160,17 +161,17 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingestSlice [] return fmt.Errorf("failed to ingest after %d retries: %s", maxRetries, tableName) } -func getTableIdFromCache(tableName string) (uuid.UUID, bool) { - tableMutex.Lock() - defer tableMutex.Unlock() - id, exists := tableCache[tableName] +func (p *HydrolixBackendConnector) getTableIdFromCache(tableName string) (uuid.UUID, bool) { + p.tableMutex.Lock() + defer p.tableMutex.Unlock() + id, exists := p.tableCache[tableName] return id, exists } -func setTableIdInCache(tableName string, tableId uuid.UUID) { - tableMutex.Lock() - defer tableMutex.Unlock() - tableCache[tableName] = tableId +func (p *HydrolixBackendConnector) setTableIdInCache(tableName string, tableId uuid.UUID) { + p.tableMutex.Lock() + defer p.tableMutex.Unlock() + p.tableCache[tableName] = tableId } func (p *HydrolixBackendConnector) createTableWithSchema(ctx context.Context, @@ -233,7 +234,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. } tableName := createTable["name"].(string) - tableId, _ := getTableIdFromCache(tableName) + tableId, _ := p.getTableIdFromCache(tableName) if len(createTable) > 0 && tableId == uuid.Nil { tableId = uuid.New() createTable["uuid"] = tableId.String() @@ -242,7 +243,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. logger.ErrorWithCtx(ctx).Msgf("error creating table with schema: %v", err) return err } - setTableIdInCache(tableName, tableId) + p.setTableIdInCache(tableName, tableId) } if len(ingestSlice) > 0 { From 955dd01c532818a086927e90b8d8af496af97491 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 28 Jul 2025 16:45:34 +0200 Subject: [PATCH 48/54] Getting rid of some hardcoded values --- .../hydrolix_backend_connector.go | 33 +++++++------------ platform/clickhouse/connection.go | 2 +- platform/config/config_v2.go | 3 ++ platform/ingest/insert_test.go | 2 +- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 019537a2a..3eaf0394f 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -22,14 +22,10 @@ import ( type HydrolixBackendConnector struct { BasicSqlBackendConnector - // TODO for now we still have reference for RelationalDbConfiguration for fallback - cfg *config.RelationalDbConfiguration - IngestURL string - AccessToken string - Headers map[string]string - client *http.Client - tableCache map[string]uuid.UUID - tableMutex sync.Mutex + cfg *config.RelationalDbConfiguration + client *http.Client + tableCache map[string]uuid.UUID + tableMutex sync.Mutex } func (p *HydrolixBackendConnector) GetId() quesma_api.BackendConnectorType { @@ -58,11 +54,12 @@ func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration } } -func NewHydrolixBackendConnectorWithConnection(_ string, conn *sql.DB) *HydrolixBackendConnector { +func NewHydrolixBackendConnectorWithConnection(configuration *config.RelationalDbConfiguration, conn *sql.DB) *HydrolixBackendConnector { return &HydrolixBackendConnector{ BasicSqlBackendConnector: BasicSqlBackendConnector{ connection: conn, }, + cfg: configuration, client: &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -110,12 +107,6 @@ func (p *HydrolixBackendConnector) makeRequest(ctx context.Context, method strin return respBody, err } -// TODO hardcoded for now -const token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIybDZyTk1YV2hYQTA5M2tkRHA5ZFctaEMzM2NkOEtWUFhJdURZLWlLeUFjIn0.eyJleHAiOjE3NTM3NzY2NTksImlhdCI6MTc1MzY5MDI1OSwianRpIjoiMzNmNzI2M2MtMTA2Zi00MTc1LWJhZTEtOTEzNTJkNTdmOWM0IiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3Qva2V5Y2xvYWsvcmVhbG1zL2h5ZHJvbGl4LXVzZXJzIiwiYXVkIjpbImNvbmZpZy1hcGkiLCJhY2NvdW50Il0sInN1YiI6ImRiMWM1YTJiLTdhYjMtNGNmZi04NGU4LTQ3Yzc0YjRlZjAyMSIsInR5cCI6IkJlYXJlciIsImF6cCI6ImNvbmZpZy1hcGkiLCJzZXNzaW9uX3N0YXRlIjoiNGRhZWM2YzItMzA4ZC00MzFkLTg0ZWMtNGFiMjJjOTFmZjg3IiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0Il0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWh5ZHJvbGl4LXVzZXJzIiwib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGNvbmZpZy1hcGktc2VydmljZSBlbWFpbCBwcm9maWxlIiwic2lkIjoiNGRhZWM2YzItMzA4ZC00MzFkLTg0ZWMtNGFiMjJjOTFmZjg3IiwiZW1haWxfdmVyaWZpZWQiOnRydWUsInByZWZlcnJlZF91c2VybmFtZSI6Im1lQGh5ZHJvbGl4LmlvIiwiZW1haWwiOiJtZUBoeWRyb2xpeC5pbyJ9.Yr0hleV6sJZCmOQKXSN82HVRm4RKC7IGW7CVXHJai8vOKMW5uPIiw_1BwaHzKi8DjwftHvhWW0hmEXh492Mj_6csQgvejeCfwbKvZx9rQbBZ-4P4GboB4OgqtZ5macY6D_QQyeXol2otS80E8OTAUBM8o07v_fYd92-nz-qY7ceicT8oI7kLMgEOD6VA7Glue7hqQblofIZMoDK1Ve2WhrOhfgqVDxCloFrLs1VhXevGBkVgz7LF_XoxLyR0UPhyVj7lM3ep3M8FJbuP5afKuJUr2nb3qm5Bxs_r1uuQe7INuEH-CYCPJmsOArJ0BIULgtB3LW1zCsLl_DAMQJhwtg" -const hdxHost = "3.20.203.177:8888" -const orgID = "d9ce0431-f26f-44e3-b0ef-abc1653d04eb" -const projectID = "27506b30-0c78-41fa-a059-048d687f1164" - type HydrolixResponse struct { Code int `json:"code"` Message string `json:"message"` @@ -144,11 +135,11 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingestSlice [] return fmt.Errorf("failed to marshal final JSON array: %w", err) } - url := fmt.Sprintf("http://%s/ingest/event", hdxHost) + url := fmt.Sprintf("%s/ingest/event", p.cfg.Url.String()) const sleepDuration = 5 * time.Second const maxRetries = 5 for retries := 0; retries < maxRetries; retries++ { - _, err := p.makeRequest(ctx, "POST", url, finalJson, token, tableName) + _, err := p.makeRequest(ctx, "POST", url, finalJson, p.cfg.Token, tableName) if err != nil { logger.WarnWithCtx(ctx).Msgf("Error ingesting table %s: %v retrying...", tableName, err) time.Sleep(sleepDuration) @@ -177,27 +168,27 @@ func (p *HydrolixBackendConnector) setTableIdInCache(tableName string, tableId u func (p *HydrolixBackendConnector) createTableWithSchema(ctx context.Context, createTable map[string]interface{}, transform map[string]interface{}, tableName string, tableId uuid.UUID) error { - url := fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/", hdxHost, orgID, projectID) + url := fmt.Sprintf("%s/config/v1/orgs/%s/projects/%s/tables/", p.cfg.Url.String(), p.cfg.OrgId, p.cfg.ProjectId) createTableJson, err := json.Marshal(createTable) logger.Info().Msgf("createtable event: %s %s", tableName, string(createTableJson)) if err != nil { return fmt.Errorf("error marshalling create_table JSON: %v", err) } - _, err = p.makeRequest(ctx, "POST", url, createTableJson, token, tableName) + _, err = p.makeRequest(ctx, "POST", url, createTableJson, p.cfg.Token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err } - url = fmt.Sprintf("http://%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", hdxHost, orgID, projectID, tableId.String()) + url = fmt.Sprintf("%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", p.cfg.Url.String(), p.cfg.OrgId, p.cfg.ProjectId, tableId.String()) transformJson, err := json.Marshal(transform) if err != nil { return fmt.Errorf("error marshalling transform JSON: %v", err) } logger.Info().Msgf("transform event: %s %s", tableName, string(transformJson)) - _, err = p.makeRequest(ctx, "POST", url, transformJson, token, tableName) + _, err = p.makeRequest(ctx, "POST", url, transformJson, p.cfg.Token, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err diff --git a/platform/clickhouse/connection.go b/platform/clickhouse/connection.go index fc6e59ea8..cb1e4080f 100644 --- a/platform/clickhouse/connection.go +++ b/platform/clickhouse/connection.go @@ -98,7 +98,7 @@ func InitDBConnectionPool(c *config.QuesmaConfiguration) quesma_api.BackendConne db.SetConnMaxLifetime(time.Duration(5) * time.Minute) // default is 1h if c.Hydrolix.ConnectorType == quesma_api.GetBackendConnectorNameFromType(quesma_api.HydrolixSQLBackend) { - return backend_connectors.NewHydrolixBackendConnectorWithConnection(c.Hydrolix.Url.String(), db) + return backend_connectors.NewHydrolixBackendConnectorWithConnection(&c.Hydrolix, db) } return backend_connectors.NewClickHouseBackendConnectorWithConnection(c.ClickHouse.Url.String(), db) diff --git a/platform/config/config_v2.go b/platform/config/config_v2.go index 92a91e046..63400c31e 100644 --- a/platform/config/config_v2.go +++ b/platform/config/config_v2.go @@ -108,6 +108,9 @@ type RelationalDbConfiguration struct { ClientCertPath string `koanf:"clientCertPath"` ClientKeyPath string `koanf:"clientKeyPath"` CACertPath string `koanf:"caCertPath"` + Token string `koanf:"token"` + OrgId string `koanf:"orgId"` + ProjectId string `koanf:"projectId"` } func (c *RelationalDbConfiguration) IsEmpty() bool { diff --git a/platform/ingest/insert_test.go b/platform/ingest/insert_test.go index a1d3ab6d7..ae10e3b7f 100644 --- a/platform/ingest/insert_test.go +++ b/platform/ingest/insert_test.go @@ -486,7 +486,7 @@ func TestHydrolixIngest(t *testing.T) { tables := NewTableMap() conn, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - db := backend_connectors.NewHydrolixBackendConnectorWithConnection("", conn) + db := backend_connectors.NewHydrolixBackendConnectorWithConnection(&quesmaConfig.Hydrolix, conn) if err != nil { t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) } From 4973f5e534213d839a5e4aabd0e19b053a3e8f36 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Mon, 28 Jul 2025 17:18:16 +0200 Subject: [PATCH 49/54] Improvements #4 --- platform/config/config_v2.go | 6 +++--- platform/licensing/runner.go | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/platform/config/config_v2.go b/platform/config/config_v2.go index 63400c31e..74f0fdf93 100644 --- a/platform/config/config_v2.go +++ b/platform/config/config_v2.go @@ -103,14 +103,14 @@ type RelationalDbConfiguration struct { ClusterName string `koanf:"clusterName"` // When creating tables by Quesma - they'll use `ON CLUSTER ClusterName` clause AdminUrl *Url `koanf:"adminUrl"` DisableTLS bool `koanf:"disableTLS"` + Token string `koanf:"token"` + OrgId string `koanf:"orgId"` + ProjectId string `koanf:"projectId"` // This supports es backend only. ClientCertPath string `koanf:"clientCertPath"` ClientKeyPath string `koanf:"clientKeyPath"` CACertPath string `koanf:"caCertPath"` - Token string `koanf:"token"` - OrgId string `koanf:"orgId"` - ProjectId string `koanf:"projectId"` } func (c *RelationalDbConfiguration) IsEmpty() bool { diff --git a/platform/licensing/runner.go b/platform/licensing/runner.go index 021273870..1a0d32ed2 100644 --- a/platform/licensing/runner.go +++ b/platform/licensing/runner.go @@ -81,9 +81,12 @@ func (l *LicenseModule) Run() { func (l *LicenseModule) validateConfig() error { // Check if connectors are allowed for _, conn := range l.Config.Connectors { + // TODO remove this once hydrolix connector is fully integrated + if conn.ConnectorType == "hydrolix" { + continue + } if !slices.Contains(l.License.Connectors, conn.ConnectorType) { - // TODO !!!!! - //return fmt.Errorf("connector of type [%s] is not allowed within the current license", conn.ConnectorType) + return fmt.Errorf("connector of type [%s] is not allowed within the current license", conn.ConnectorType) } } return nil From ac7e00505614b902f91fc86dccd8e2cd7a6d65b7 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 29 Jul 2025 13:37:12 +0200 Subject: [PATCH 50/54] Returning error instead of panic --- .../backend_connectors/hydrolix_backend_connector.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 3eaf0394f..6cc74c5e6 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -83,7 +83,7 @@ func (p *HydrolixBackendConnector) makeRequest(ctx context.Context, method strin // Build the request req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body)) if err != nil { - panic(err) + return nil, err } // Set headers @@ -206,7 +206,7 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. // Top-level object var root map[string]json.RawMessage if err := json.Unmarshal([]byte(query), &root); err != nil { - panic(err) + return err } // Extract each section into its own map (or struct, if needed) @@ -215,13 +215,13 @@ func (p *HydrolixBackendConnector) Exec(_ context.Context, query string, args .. var ingestSlice []map[string]interface{} if err := json.Unmarshal(root["create_table"], &createTable); err != nil { - panic(err) + return err } if err := json.Unmarshal(root["transform"], &transform); err != nil { - panic(err) + return err } if err := json.Unmarshal(root["ingest"], &ingestSlice); err != nil { - panic(err) + return err } tableName := createTable["name"].(string) From a896c63b5cffff4701b97af289685e3fabe49af5 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 29 Jul 2025 13:43:13 +0200 Subject: [PATCH 51/54] Review remarks #2 --- platform/backend_connectors/hydrolix_backend_connector.go | 2 ++ platform/ingest/hydrolixlowerer.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 6cc74c5e6..f5b9e31b9 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -136,6 +136,8 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingestSlice [] } url := fmt.Sprintf("%s/ingest/event", p.cfg.Url.String()) + // Sleep duration is arbitrarily chosen. + // It seems that the Hydrolix API needs some time to process the table creation before ingesting data. const sleepDuration = 5 * time.Second const maxRetries = 5 for retries := 0; retries < maxRetries; retries++ { diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 0552fc8fb..6760df535 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -26,6 +26,7 @@ type HydrolixLowerer struct { ingestFieldStatistics IngestFieldStatistics ingestFieldStatisticsLock sync.Mutex tableCreteStatementMapping map[*chLib.Table]CreateTableStatement // cache for table creation statements + tableCreationLock sync.Mutex } func NewHydrolixLowerer(virtualTableStorage persistence.JSONDatabase) *HydrolixLowerer { @@ -451,11 +452,13 @@ func (l *HydrolixLowerer) LowerToDDL( createTableCmd CreateTableStatement, ) ([]string, error) { + l.tableCreationLock.Lock() if _, exists := l.tableCreteStatementMapping[table]; !exists { l.tableCreteStatementMapping[table] = createTableCmd } else { createTableCmd = l.tableCreteStatementMapping[table] } + l.tableCreationLock.Unlock() // --- Create Table Section --- createTable := map[string]interface{}{ From 82035034573931e193247f3eadeeffd740970ae7 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 29 Jul 2025 13:44:27 +0200 Subject: [PATCH 52/54] Review remarks #3 --- platform/ingest/hydrolixlowerer.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/platform/ingest/hydrolixlowerer.go b/platform/ingest/hydrolixlowerer.go index 6760df535..98976846c 100644 --- a/platform/ingest/hydrolixlowerer.go +++ b/platform/ingest/hydrolixlowerer.go @@ -639,6 +639,9 @@ func (l *HydrolixLowerer) LowerToDDL( } } // --- Final Payload --- + // There is implicit interface here between lowerer and backend connector + // so we need to generate payload that is compatible with backend connector + // backend connector expects a specific structure payload := map[string]interface{}{ "create_table": createTable, "transform": transform, From f0e3f88b57ec6616b8291e40c5645ee904aafdc0 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 29 Jul 2025 13:47:34 +0200 Subject: [PATCH 53/54] Adding Hydrolix prefix --- .../hydrolix_backend_connector.go | 10 ++++----- platform/config/config_v2.go | 22 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index f5b9e31b9..0d76293ce 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -141,7 +141,7 @@ func (p *HydrolixBackendConnector) ingestFun(ctx context.Context, ingestSlice [] const sleepDuration = 5 * time.Second const maxRetries = 5 for retries := 0; retries < maxRetries; retries++ { - _, err := p.makeRequest(ctx, "POST", url, finalJson, p.cfg.Token, tableName) + _, err := p.makeRequest(ctx, "POST", url, finalJson, p.cfg.HydrolixToken, tableName) if err != nil { logger.WarnWithCtx(ctx).Msgf("Error ingesting table %s: %v retrying...", tableName, err) time.Sleep(sleepDuration) @@ -170,27 +170,27 @@ func (p *HydrolixBackendConnector) setTableIdInCache(tableName string, tableId u func (p *HydrolixBackendConnector) createTableWithSchema(ctx context.Context, createTable map[string]interface{}, transform map[string]interface{}, tableName string, tableId uuid.UUID) error { - url := fmt.Sprintf("%s/config/v1/orgs/%s/projects/%s/tables/", p.cfg.Url.String(), p.cfg.OrgId, p.cfg.ProjectId) + url := fmt.Sprintf("%s/config/v1/orgs/%s/projects/%s/tables/", p.cfg.Url.String(), p.cfg.HydrolixOrgId, p.cfg.HydrolixProjectId) createTableJson, err := json.Marshal(createTable) logger.Info().Msgf("createtable event: %s %s", tableName, string(createTableJson)) if err != nil { return fmt.Errorf("error marshalling create_table JSON: %v", err) } - _, err = p.makeRequest(ctx, "POST", url, createTableJson, p.cfg.Token, tableName) + _, err = p.makeRequest(ctx, "POST", url, createTableJson, p.cfg.HydrolixToken, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err } - url = fmt.Sprintf("%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", p.cfg.Url.String(), p.cfg.OrgId, p.cfg.ProjectId, tableId.String()) + url = fmt.Sprintf("%s/config/v1/orgs/%s/projects/%s/tables/%s/transforms", p.cfg.Url.String(), p.cfg.HydrolixOrgId, p.cfg.HydrolixProjectId, tableId.String()) transformJson, err := json.Marshal(transform) if err != nil { return fmt.Errorf("error marshalling transform JSON: %v", err) } logger.Info().Msgf("transform event: %s %s", tableName, string(transformJson)) - _, err = p.makeRequest(ctx, "POST", url, transformJson, p.cfg.Token, tableName) + _, err = p.makeRequest(ctx, "POST", url, transformJson, p.cfg.HydrolixToken, tableName) if err != nil { logger.ErrorWithCtx(ctx).Msgf("error making request: %v", err) return err diff --git a/platform/config/config_v2.go b/platform/config/config_v2.go index 74f0fdf93..bb78e69c6 100644 --- a/platform/config/config_v2.go +++ b/platform/config/config_v2.go @@ -95,17 +95,17 @@ type BackendConnector struct { // RelationalDbConfiguration works fine for non-relational databases too, consider rename type RelationalDbConfiguration struct { //ConnectorName string `koanf:"name"` - ConnectorType string `koanf:"type"` - Url *Url `koanf:"url"` - User string `koanf:"user"` - Password string `koanf:"password"` - Database string `koanf:"database"` - ClusterName string `koanf:"clusterName"` // When creating tables by Quesma - they'll use `ON CLUSTER ClusterName` clause - AdminUrl *Url `koanf:"adminUrl"` - DisableTLS bool `koanf:"disableTLS"` - Token string `koanf:"token"` - OrgId string `koanf:"orgId"` - ProjectId string `koanf:"projectId"` + ConnectorType string `koanf:"type"` + Url *Url `koanf:"url"` + User string `koanf:"user"` + Password string `koanf:"password"` + Database string `koanf:"database"` + ClusterName string `koanf:"clusterName"` // When creating tables by Quesma - they'll use `ON CLUSTER ClusterName` clause + AdminUrl *Url `koanf:"adminUrl"` + DisableTLS bool `koanf:"disableTLS"` + HydrolixToken string `koanf:"token"` + HydrolixOrgId string `koanf:"orgId"` + HydrolixProjectId string `koanf:"projectId"` // This supports es backend only. ClientCertPath string `koanf:"clientCertPath"` From f0a187956553ccdf0308c39b8317a7d9f483a617 Mon Sep 17 00:00:00 2001 From: Przemek Delewski Date: Tue, 29 Jul 2025 13:55:55 +0200 Subject: [PATCH 54/54] Config validation --- .../hydrolix_backend_connector.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/platform/backend_connectors/hydrolix_backend_connector.go b/platform/backend_connectors/hydrolix_backend_connector.go index 0d76293ce..1b13f375d 100644 --- a/platform/backend_connectors/hydrolix_backend_connector.go +++ b/platform/backend_connectors/hydrolix_backend_connector.go @@ -41,7 +41,27 @@ func (p *HydrolixBackendConnector) Open() error { return nil } +func checkHydrolixConfig(cfg *config.RelationalDbConfiguration) error { + if cfg.Url == nil { + return fmt.Errorf("hydrolix URL is not set") + } + if cfg.HydrolixToken == "" { + return fmt.Errorf("hydrolix token is not set") + } + if cfg.HydrolixOrgId == "" { + return fmt.Errorf("hydrolix organization ID is not set") + } + if cfg.HydrolixProjectId == "" { + return fmt.Errorf("hydrolix project ID is not set") + } + return nil +} + func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration) *HydrolixBackendConnector { + if err := checkHydrolixConfig(configuration); err != nil { + logger.Error().Msgf("Invalid Hydrolix configuration: %v", err) + return nil + } return &HydrolixBackendConnector{ cfg: configuration, client: &http.Client{ @@ -55,6 +75,10 @@ func NewHydrolixBackendConnector(configuration *config.RelationalDbConfiguration } func NewHydrolixBackendConnectorWithConnection(configuration *config.RelationalDbConfiguration, conn *sql.DB) *HydrolixBackendConnector { + if err := checkHydrolixConfig(configuration); err != nil { + logger.Error().Msgf("Invalid Hydrolix configuration: %v", err) + return nil + } return &HydrolixBackendConnector{ BasicSqlBackendConnector: BasicSqlBackendConnector{ connection: conn,