diff --git a/ci/it/configs/quesma-now-reads-clickhouse-tables.yml.template b/ci/it/configs/quesma-now-reads-clickhouse-tables.yml.template index bd516af9e..2c15823e4 100644 --- a/ci/it/configs/quesma-now-reads-clickhouse-tables.yml.template +++ b/ci/it/configs/quesma-now-reads-clickhouse-tables.yml.template @@ -25,9 +25,11 @@ processors: config: indexes: test_table: - target: [ my-clickhouse-instance] + target: + - my-clickhouse-instance "*": - target: [my-minimal-elasticsearch] + target: + - my-minimal-elasticsearch pipelines: - name: my-elasticsearch-proxy-read frontendConnectors: [ elastic-query ] diff --git a/ci/it/configs/quesma-wildcard-clickhouse.yml.template b/ci/it/configs/quesma-wildcard-clickhouse.yml.template index 514c93b34..e650db443 100644 --- a/ci/it/configs/quesma-wildcard-clickhouse.yml.template +++ b/ci/it/configs/quesma-wildcard-clickhouse.yml.template @@ -23,15 +23,16 @@ processors: - name: my-query-processor type: quesma-v1-processor-query config: - indexes: + indexes: "*": - target: [ my-clickhouse-instance ] + target: + - my-clickhouse-instance - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: "*": - target: [ my-clickhouse-instance ] + target: [ my-clickhouse-instance ] # deliberately using legacy way of listing targets here (for better coverage) pipelines: - name: my-elasticsearch-proxy-read frontendConnectors: [ elastic-query ] diff --git a/ci/it/configs/quesma-wildcard-disabled.yml.template b/ci/it/configs/quesma-wildcard-disabled.yml.template index 3587e2721..7589396c4 100644 --- a/ci/it/configs/quesma-wildcard-disabled.yml.template +++ b/ci/it/configs/quesma-wildcard-disabled.yml.template @@ -25,25 +25,27 @@ processors: config: indexes: "explicitly_disabled1": - target: [ ] + target: "explicitly_disabled3": - target: [ ] + target: "query_enabled": - target: [ my-clickhouse-instance ] + target: + - my-clickhouse-instance "*": - target: [ ] + target: - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: "explicitly_disabled2": - target: [ ] + target: "explicitly_disabled3": - target: [ ] + target: "ingest_enabled": - target: [ my-clickhouse-instance ] + target: + - my-clickhouse-instance "*": - target: [ ] + target: pipelines: - name: my-elasticsearch-proxy-read frontendConnectors: [ elastic-query ] @@ -53,4 +55,3 @@ pipelines: frontendConnectors: [ elastic-ingest ] processors: [ my-ingest-processor ] backendConnectors: [ my-minimal-elasticsearch, my-clickhouse-instance ] - diff --git a/ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template b/ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template index a33e73db3..d880750a4 100644 --- a/ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template +++ b/ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template @@ -25,14 +25,20 @@ processors: config: indexes: logs-1: - target: [ e ] + target: + - e logs-2: - target: [ c ] + target: + - c logs-3: - target: [ c, e ] + target: + - c + - e ## WARNING `logs-dual-query` (with two dashes) falls under default index pattern for logs in Elasticsearch and results in not index, but datastream creation logs-dual_query: - target: [ c, e ] + target: + - c + - e logs-4: target: - c: @@ -42,19 +48,26 @@ processors: - c: useCommonTable: true "*": - target: [ e ] + target: + - e - name: IP type: quesma-v1-processor-ingest config: indexes: logs-1: - target: [ e ] + target: + - e logs-2: - target: [ c ] + target: + - c logs-3: - target: [ c, e ] + target: + - c + - e logs-dual_query: - target: [ c, e ] + target: + - c + - e logs-4: target: - c: @@ -62,7 +75,8 @@ processors: logs-5: target: "*": - target: [ e ] + target: + - e pipelines: - name: my-elasticsearch-proxy-read @@ -72,4 +86,4 @@ pipelines: - name: my-elasticsearch-proxy-write frontendConnectors: [ elastic-ingest ] processors: [ IP ] - backendConnectors: [ e, c ] + backendConnectors: [ e, c ] \ No newline at end of file diff --git a/ci/it/configs/quesma-with-two-pipelines.yml.template b/ci/it/configs/quesma-with-two-pipelines.yml.template index 9a0ae7077..b9fec1cee 100644 --- a/ci/it/configs/quesma-with-two-pipelines.yml.template +++ b/ci/it/configs/quesma-with-two-pipelines.yml.template @@ -25,23 +25,29 @@ processors: config: indexes: siem: - target: [my-hydrolix-instance] + target: + - my-hydrolix-instance logs: - target: [my-hydrolix-instance] + target: + - my-hydrolix-instance test_index: - target: [my-minimal-elasticsearch] + target: + - my-minimal-elasticsearch "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: test_index: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch logs_disabled: - target: [ ] + target: "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: my-elasticsearch-proxy-read frontendConnectors: [ elastic-query ] @@ -51,4 +57,3 @@ pipelines: frontendConnectors: [ elastic-ingest ] processors: [ my-ingest-processor ] backendConnectors: [ my-minimal-elasticsearch, my-hydrolix-instance ] - diff --git a/ci/quesma/config/ci-config.yaml b/ci/quesma/config/ci-config.yaml index 2c98e5a5c..12f2bf2dd 100644 --- a/ci/quesma/config/ci-config.yaml +++ b/ci/quesma/config/ci-config.yaml @@ -24,41 +24,53 @@ processors: config: indexes: kibana_sample_data_ecommerce: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_flights: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source logs-generic-default: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: "message": type: text windows_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: p2 type: quesma-v1-processor-ingest config: indexes: kibana_sample_data_ecommerce: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_flights: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source logs-generic-default: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: "message": type: text windows_logs: # Used for EQL e2e tests - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: p-query diff --git a/docs/public/docs/compatibility-report.md b/docs/public/docs/compatibility-report.md index 0b00a26da..c0da5f1f4 100644 --- a/docs/public/docs/compatibility-report.md +++ b/docs/public/docs/compatibility-report.md @@ -17,15 +17,19 @@ processors: config: indexes: kibana_sample_data_ecommerce: - target: [ backend-elastic, backend-clickhouse ] + target: + - backend-elastic + - backend-clickhouse ab_testing_logs: - target: [ backend-clickhouse ] + target: + - backend-clickhouse - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: ab_testing_logs: - target: [ backend-clickhouse ] + target: + - backend-clickhouse ``` This configuration turns on Compatibility Report for `kibana_sample_data_ecommerce` index. The `ab_testing_logs` is an internal Quesma index which is required in the configuration for Compatibility Report to work properly. diff --git a/docs/public/docs/config-primer.md b/docs/public/docs/config-primer.md index ed910e07f..a299c469a 100644 --- a/docs/public/docs/config-primer.md +++ b/docs/public/docs/config-primer.md @@ -120,7 +120,8 @@ processors: config: indexes: kibana_sample_data_ecommerce: - target: [ backend-clickhouse ] + target: + - backend-clickhouse schemaOverrides: fields: "geoip.location": @@ -128,7 +129,8 @@ processors: "products.product_name": type: text "*": - target: [ backend-elastic ] + target: + - backend-elastic ``` To get more specific information on processor configuration, please refer to [Processor configuration](#processor-configuration) section. @@ -168,15 +170,18 @@ processors: config: indexes: kibana_sample_data_logs: - target: [ backend-elasticsearch ] + target: + - backend-elasticsearch kibana_sample_data_ecommerce: - target: [ backend-clickhouse ] + target: + - backend-clickhouse schemaOverrides: fields: "geoip.location": type: geo_point "*": # Always required - target: [ backend-elasticsearch ] + target: + - backend-elasticsearch ``` ### Index configuration @@ -187,11 +192,13 @@ The configuration for an index consists of the following configuration options: - `target` (required): a list of backend connectors that will handle the request. For example the following configuration in the ingest processor: ```yaml my_index: - target: [ backend-elasticsearch, backend-clickhouse ] + target: + - backend-elasticsearch + - backend-clickhouse ``` will dual write ingest requests to `my_index` to both ElasticSearch and ClickHouse. Note that ElasticSearch/OpenSearch is the only supported backend for the `*` entry. - If no targets are provided (example: `target: []`) in the configuration of an index in the ingest processor, ingest for that index will be disabled and incoming data will be dropped. + If no targets are provided (empty `target` list) in the configuration of an index in the ingest processor, ingest for that index will be disabled and incoming data will be dropped. For the query processor by specifing multiple targets, [Compatibility report](/compatibility-report.md) will be enabled. See [Compatibility report documentation](/compatibility-report.md) for more details. Some backend connectors have additional attributes which may be used. For example the following configuration sets `useCommonTable` for `backend-clickhouse` target: @@ -207,7 +214,8 @@ The configuration for an index consists of the following configuration options: - `schemaOverrides` (optional): manual overrides of schema information for an index. Quesma infers schema for an index based on the data ingested and the schema information fetched from ClickHouse. `schemaOverrides` allows you to override this inferred schema with for some fields. For example the following configuration: ```yaml my_index: - target: [ backend-clickhouse ] + target: + - backend-clickhouse schemaOverrides: "product_name": type: "text" diff --git a/docs/public/docs/example-2-0-clickhouse-specific.md b/docs/public/docs/example-2-0-clickhouse-specific.md index 4154438e9..5d3dc8394 100644 --- a/docs/public/docs/example-2-0-clickhouse-specific.md +++ b/docs/public/docs/example-2-0-clickhouse-specific.md @@ -72,21 +72,27 @@ flowchart LR config: indexes: # `siem` and `logs` are just examples, siem: # make sure to replace them with your actual table names in your ClickHouse instance - target: [ clickhouse-instance ] + target: + - clickhouse-instance logs: - target: [ clickhouse-instance ] + target: + - clickhouse-instance '*': # DO NOT remove, always required - target: [ minimal-elasticsearch ] + target: + - minimal-elasticsearch - name: ingest-processor type: quesma-v1-processor-ingest config: indexes: # `siem` and `logs` are just examples, siem: # make sure to replace them with your actual table or index names in your ClickHouse instance - target: [ clickhouse-instance ] + target: + - clickhouse-instance logs: - target: [ clickhouse-instance ] + target: + - clickhouse-instance '*': # DO NOT remove, always required - target: [ minimal-elasticsearch ] + target: + - minimal-elasticsearch pipelines: - name: elasticsearch-proxy-read frontendConnectors: [ elastic-query ] diff --git a/docs/public/docs/example-2-1-hydro-specific.md b/docs/public/docs/example-2-1-hydro-specific.md index 8ec4b724c..05832a3bb 100644 --- a/docs/public/docs/example-2-1-hydro-specific.md +++ b/docs/public/docs/example-2-1-hydro-specific.md @@ -60,29 +60,39 @@ flowchart LR config: indexes: # the list below is just an example, siem: # make sure to replace them with your actual table or index names - target: [ hydrolix-instance ] + target: + - hydrolix-instance logs: - target: [ hydrolix-instance ] + target: + - hydrolix-instance index1: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance index2: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance index3: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance '*': # DO NOT remove, always required - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance - name: ingest-processor type: quesma-v1-processor-ingest config: indexes: # the list below is just an example, index1: # make sure to replace them with your actual table or index names - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance index2: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance index3: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance '*': # DO NOT remove, always required - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance pipelines: - name: elasticsearch-proxy-read frontendConnectors: [ elastic-query ] diff --git a/docs/public/docs/example-2-1.md b/docs/public/docs/example-2-1.md index 5cd88a948..c7878e486 100644 --- a/docs/public/docs/example-2-1.md +++ b/docs/public/docs/example-2-1.md @@ -69,33 +69,45 @@ K[Kibana/OSD] --> Q((Quesma)) config: indexes: # the list below is just an example, siem: # make sure to replace them with your actual table or index names - target: [ clickhouse-instance ] + target: + - clickhouse-instance logs: - target: [ clickhouse-instance ] + target: + - clickhouse-instance index1: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance index2: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance index3: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance '*': # DO NOT remove, always required - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance - name: ingest-processor type: quesma-v1-processor-ingest config: indexes: # the list below is just an example, siem: # make sure to replace them with your actual table or index names - target: [ clickhouse-instance ] + target: + - clickhouse-instance logs: - target: [ clickhouse-instance ] + target: + - clickhouse-instance index1: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance index2: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance index3: - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance '*': # DO NOT remove, always required - target: [ elasticsearch-instance ] + target: + - elasticsearch-instance pipelines: - name: elasticsearch-proxy-read frontendConnectors: [ elastic-query ] diff --git a/docs/public/docs/ingest.md b/docs/public/docs/ingest.md index 90821fcf3..6679c2176 100644 --- a/docs/public/docs/ingest.md +++ b/docs/public/docs/ingest.md @@ -23,9 +23,11 @@ processors: config: indexes: my_index: - target: [ backend-clickhouse ] + target: + - backend-clickhouse "*": # Always required - target: [ backend-elasticsearch ] + target: + - backend-elasticsearch ``` This way, Quesma will know that the data of that index should be ingested to ClickHouse. @@ -71,7 +73,8 @@ You can also specify the explicit mappings in the Quesma configuration file with ```yaml my_index: - target: [ backend-clickhouse ] + target: + - backend-clickhouse schemaOverrides: fields: "products.manufacturer": @@ -108,7 +111,8 @@ indexes: - backend-clickhouse: useCommonTable: true "*": - target: [ backend-elastic ] + target: + - backend-elastic ... ``` diff --git a/examples/kibana-sample-data/quesma/config/local-dev.yaml b/examples/kibana-sample-data/quesma/config/local-dev.yaml index c973d24c6..f20e9de5f 100644 --- a/examples/kibana-sample-data/quesma/config/local-dev.yaml +++ b/examples/kibana-sample-data/quesma/config/local-dev.yaml @@ -31,7 +31,8 @@ processors: config: indexes: kibana_sample_data_ecommerce: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: "geoip.location": @@ -45,7 +46,8 @@ processors: manufacturer: type: text kibana_sample_data_flights: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: "DestLocation": @@ -53,7 +55,8 @@ processors: "OriginLocation": type: geo_point logs-generic-default: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: timestamp: @@ -70,13 +73,15 @@ processors: severity: type: "keyword" "*": # DO NOT remove, always required - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: kibana_sample_data_ecommerce: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: "geoip.location": @@ -90,7 +95,8 @@ processors: manufacturer: type: text kibana_sample_data_flights: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: "DestLocation": @@ -98,7 +104,8 @@ processors: "OriginLocation": type: geo_point logs-generic-default: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: timestamp: @@ -115,7 +122,8 @@ processors: severity: type: "keyword" "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: my-pipeline-elasticsearch-query-clickhouse frontendConnectors: [ elastic-query ] diff --git a/quesma/quesma/config/config_v2_test.go b/quesma/quesma/config/config_v2_test.go index bf20feeef..e0b7f9ff1 100644 --- a/quesma/quesma/config/config_v2_test.go +++ b/quesma/quesma/config/config_v2_test.go @@ -268,6 +268,35 @@ func TestTargetNewVariant(t *testing.T) { assert.Equal(t, expectedOverride, override.Override) } +func TestTargetLegacyVariant(t *testing.T) { + os.Setenv(configFileLocationEnvVar, "./test_configs/target_legacy_variant.yaml") + cfg := LoadV2Config() + if err := cfg.Validate(); err != nil { + t.Fatalf("error validating config: %v", err) + } + legacyConf := cfg.TranslateToLegacyConfig() + assert.False(t, legacyConf.TransparentProxy) + assert.Equal(t, 3, len(legacyConf.IndexConfig)) + ecommerce := legacyConf.IndexConfig["kibana_sample_data_ecommerce"] + flights := legacyConf.IndexConfig["kibana_sample_data_flights"] + logs := legacyConf.IndexConfig["kibana_sample_data_logs"] + + assert.Equal(t, []string{ClickhouseTarget}, ecommerce.QueryTarget) + assert.Equal(t, []string{ClickhouseTarget, ElasticsearchTarget}, ecommerce.IngestTarget) + + assert.Equal(t, []string{ClickhouseTarget}, flights.QueryTarget) + assert.Equal(t, []string{ClickhouseTarget}, flights.IngestTarget) + + assert.Equal(t, []string{ElasticsearchTarget, ClickhouseTarget}, logs.QueryTarget) + assert.Equal(t, []string{ClickhouseTarget, ElasticsearchTarget}, logs.IngestTarget) + + assert.Equal(t, false, flights.UseCommonTable) + assert.Equal(t, "", flights.Override) + assert.Equal(t, false, ecommerce.UseCommonTable) + assert.Equal(t, "", ecommerce.Override) + assert.Equal(t, true, legacyConf.EnableIngest) +} + func TestUseCommonTableGlobalProperty(t *testing.T) { os.Setenv(configFileLocationEnvVar, "./test_configs/use_common_table_global_property.yaml") cfg := LoadV2Config() diff --git a/quesma/quesma/config/test_configs/has_common_table.yaml b/quesma/quesma/config/test_configs/has_common_table.yaml index 6bc80de57..ea2001174 100644 --- a/quesma/quesma/config/test_configs/has_common_table.yaml +++ b/quesma/quesma/config/test_configs/has_common_table.yaml @@ -26,11 +26,15 @@ processors: config: indexes: logs-1: - target: [ E ] + target: + - E logs-2: - target: [ E ] + target: + - E logs-3: - target: [ C, E ] + target: + - C + - E logs-4: target: - C: @@ -38,24 +42,30 @@ processors: logs-5: target: "*": - target: [ E ] + target: + - E - name: IP type: quesma-v1-processor-ingest config: indexes: logs-1: - target: [ E ] + target: + - E logs-2: - target: [ E ] + target: + - E logs-3: - target: [ C, E ] + target: + - C + - E logs-4: target: - C: useCommonTable: true "*": - target: [ E ] + target: + - E logs-5: target: diff --git a/quesma/quesma/config/test_configs/ingest_with_single_connector.yaml b/quesma/quesma/config/test_configs/ingest_with_single_connector.yaml index 72a4178e1..4257fbb43 100644 --- a/quesma/quesma/config/test_configs/ingest_with_single_connector.yaml +++ b/quesma/quesma/config/test_configs/ingest_with_single_connector.yaml @@ -33,21 +33,27 @@ processors: config: indexes: siem: - target: [my-hydrolix-instance] + target: + - my-hydrolix-instance logs: - target: [my-hydrolix-instance] + target: + - my-hydrolix-instance "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: siem: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch logs: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: my-elasticsearch-proxy-read frontendConnectors: [ elastic-query ] diff --git a/quesma/quesma/config/test_configs/invalid_dual_target.yaml b/quesma/quesma/config/test_configs/invalid_dual_target.yaml index c1318dc11..248ac6fed 100644 --- a/quesma/quesma/config/test_configs/invalid_dual_target.yaml +++ b/quesma/quesma/config/test_configs/invalid_dual_target.yaml @@ -26,21 +26,27 @@ processors: config: indexes: invalid: - target: [ E,C ] + target: + - E + - C "*": - target: [ E ] + target: + - E - name: IP type: quesma-v1-processor-ingest config: indexes: invalid: - target: [ C, E ] + target: + - C + - E "*": - target: [ E ] + target: + - E logs-5: useCommonTable: true - target: [ ] + target: pipelines: - name: my-elasticsearch-proxy-read diff --git a/quesma/quesma/config/test_configs/quesma_adding_two_hydrolix_tables.yaml b/quesma/quesma/config/test_configs/quesma_adding_two_hydrolix_tables.yaml index 9013ded1b..91ad79bea 100644 --- a/quesma/quesma/config/test_configs/quesma_adding_two_hydrolix_tables.yaml +++ b/quesma/quesma/config/test_configs/quesma_adding_two_hydrolix_tables.yaml @@ -39,21 +39,27 @@ processors: config: indexes: siem: - target: [my-hydrolix-instance] + target: + - my-hydrolix-instance logs: - target: [my-hydrolix-instance] + target: + - my-hydrolix-instance "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: # This is also virtual requirement, but it is what it is for now siem: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch logs: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: my-elasticsearch-proxy-read frontendConnectors: [ elastic-query ] diff --git a/quesma/quesma/config/test_configs/quesma_as_transparent_proxy_without_noop.yml b/quesma/quesma/config/test_configs/quesma_as_transparent_proxy_without_noop.yml index b58d48f1b..1e19090e9 100644 --- a/quesma/quesma/config/test_configs/quesma_as_transparent_proxy_without_noop.yml +++ b/quesma/quesma/config/test_configs/quesma_as_transparent_proxy_without_noop.yml @@ -30,21 +30,27 @@ processors: config: indexes: siem: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch logs: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: siem: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch logs: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: my-elasticsearch-transparent-proxy-read frontendConnectors: [ elastic-query ] diff --git a/quesma/quesma/config/test_configs/quesma_hydrolix_tables_query_only.yaml b/quesma/quesma/config/test_configs/quesma_hydrolix_tables_query_only.yaml index 923f2a88a..a653c4c76 100644 --- a/quesma/quesma/config/test_configs/quesma_hydrolix_tables_query_only.yaml +++ b/quesma/quesma/config/test_configs/quesma_hydrolix_tables_query_only.yaml @@ -39,11 +39,14 @@ processors: config: indexes: siem: - target: [my-hydrolix-instance] + target: + - my-hydrolix-instance logs: - target: [my-hydrolix-instance] + target: + - my-hydrolix-instance "*": - target: [my-minimal-elasticsearch] + target: + - my-minimal-elasticsearch pipelines: - name: my-elasticsearch-proxy-read frontendConnectors: [ elastic-query ] diff --git a/quesma/quesma/config/test_configs/target_legacy_variant.yaml b/quesma/quesma/config/test_configs/target_legacy_variant.yaml new file mode 100644 index 000000000..01ab008db --- /dev/null +++ b/quesma/quesma/config/test_configs/target_legacy_variant.yaml @@ -0,0 +1,61 @@ +# TEST CONFIGURATION +licenseKey: "cdd749a3-e777-11ee-bcf8-0242ac150004" +frontendConnectors: + - name: elastic-ingest + type: elasticsearch-fe-ingest + config: + listenPort: 8080 + - name: elastic-query + type: elasticsearch-fe-query + config: + listenPort: 8080 +backendConnectors: + - name: my-minimal-elasticsearch + type: elasticsearch + config: + url: "http://localhost:9200" + - name: my-clickhouse-data-source + type: clickhouse-os + config: + url: "clickhouse://localhost:9000" +ingestStatistics: true +internalTelemetryUrl: "https://api.quesma.com/phone-home" +logging: + remoteUrl: "https://api.quesma.com/phone-home" + path: "logs" + level: "info" +processors: + - name: my-query-processor + type: quesma-v1-processor-query + config: + indexes: + kibana_sample_data_ecommerce: + target: [my-clickhouse-data-source] + kibana_sample_data_flights: + target: [my-clickhouse-data-source] + kibana_sample_data_logs: + target: [my-minimal-elasticsearch, my-clickhouse-data-source] + "*": + target: [my-minimal-elasticsearch] + + - name: my-ingest-processor + type: quesma-v1-processor-ingest + config: + indexes: + kibana_sample_data_ecommerce: + target: [my-clickhouse-data-source, my-minimal-elasticsearch] + kibana_sample_data_flights: + target: [my-clickhouse-data-source] + kibana_sample_data_logs: + target: [my-clickhouse-data-source, my-minimal-elasticsearch] + "*": + target: [my-minimal-elasticsearch] +pipelines: + - name: my-pipeline-elasticsearch-query-clickhouse + frontendConnectors: [ elastic-query ] + processors: [ my-query-processor ] + backendConnectors: [ my-minimal-elasticsearch, my-clickhouse-data-source ] + - name: my-pipeline-elasticsearch-ingest-to-clickhouse + frontendConnectors: [ elastic-ingest ] + processors: [ my-ingest-processor ] + backendConnectors: [ my-minimal-elasticsearch, my-clickhouse-data-source ] \ No newline at end of file diff --git a/quesma/quesma/config/test_configs/target_new_variant.yaml b/quesma/quesma/config/test_configs/target_new_variant.yaml index 8f907518a..b52a130a9 100644 --- a/quesma/quesma/config/test_configs/target_new_variant.yaml +++ b/quesma/quesma/config/test_configs/target_new_variant.yaml @@ -45,7 +45,8 @@ processors: - my-clickhouse-data-source: tableName: "new_override" "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: my-ingest-processor type: quesma-v1-processor-ingest @@ -67,7 +68,8 @@ processors: - my-clickhouse-data-source: tableName: "new_override" "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: my-pipeline-elasticsearch-query-clickhouse frontendConnectors: [ elastic-query ] diff --git a/quesma/quesma/config/test_configs/test_config_v2.yaml b/quesma/quesma/config/test_configs/test_config_v2.yaml index 07785567a..ac228c6e7 100644 --- a/quesma/quesma/config/test_configs/test_config_v2.yaml +++ b/quesma/quesma/config/test_configs/test_config_v2.yaml @@ -30,24 +30,31 @@ processors: config: indexes: example-elastic-index: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch example-index: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_ecommerce: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_flights: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: timestamp: type: alias targetColumnName: "@timestamp" kafka-example-topic: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source logs-generic-default: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: message: @@ -55,50 +62,62 @@ processors: "host.name": type: text device-logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: message: type: text phone_home_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: message: type: text windows_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source phone_home_data: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: message: type: text "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: my-ingest-processor type: quesma-v1-processor-ingest config: indexes: example-elastic-index: - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch example-index: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_ecommerce: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_flights: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source kibana_sample_data_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: timestamp: type: alias targetColumnName: "@timestamp" kafka-example-topic: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source logs-generic-default: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: message: @@ -106,27 +125,32 @@ processors: "host.name": type: text device-logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: message: type: text phone_home_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: message: type: text windows_logs: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source phone_home_data: - target: [ my-clickhouse-data-source ] + target: + - my-clickhouse-data-source schemaOverrides: fields: message: type: text "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: my-pipeline-elasticsearch-query-clickhouse frontendConnectors: [ elastic-query ] diff --git a/quesma/quesma/config/test_configs/use_common_table_global_property.yaml b/quesma/quesma/config/test_configs/use_common_table_global_property.yaml index 789697e0d..ab4836e79 100644 --- a/quesma/quesma/config/test_configs/use_common_table_global_property.yaml +++ b/quesma/quesma/config/test_configs/use_common_table_global_property.yaml @@ -38,7 +38,8 @@ processors: target: - my-clickhouse-data-source "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch - name: my-ingest-processor type: quesma-v1-processor-ingest @@ -53,7 +54,8 @@ processors: target: - my-clickhouse-data-source "*": - target: [ my-minimal-elasticsearch ] + target: + - my-minimal-elasticsearch pipelines: - name: my-pipeline-elasticsearch-query-clickhouse frontendConnectors: [ elastic-query ]