Adding some integration tests (#876)

These should really boost our confidence when making changes, esp. w.r.t
routing and configuration (but not only of course 😉)

Full suite run on GitHub CI lasts <4 minutes. Locally <2 minutes given
all the images are available.

Author: mieciu

.github/workflows/integration-tests.yml

@@ -57,7 +57,7 @@ jobs:
         with:
           channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
           slack-message: |
-            :exclamation: *Integration tests failed.* :exclamation: @channel
+            :exclamation: *Integration tests failed.* :exclamation: <!channel>
             Last commit by: ${{ steps.get_author.outputs.author }}
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template

@@ -0,0 +1,72 @@
+frontendConnectors:
+  - name: elastic-ingest
+    type: elasticsearch-fe-ingest
+    config:
+      listenPort: 8080
+  - name: elastic-query
+    type: elasticsearch-fe-query
+    config:
+      listenPort: 8080
+backendConnectors:
+  - name: e
+    type: elasticsearch
+    config:
+      url: "http://{{ .elasticsearch_host }}:{{ .elasticsearch_port }}"
+      user: elastic
+      password: quesmaquesma
+  - name: c
+    type: clickhouse-os
+    config:
+      url: clickhouse://{{ .clickhouse_host }}:{{ .clickhouse_port }}
+ingestStatistics: true
+processors:
+  - name: QP
+    type: quesma-v1-processor-query
+    config:
+      indexes:
+        logs-1:
+          target: [ e ]
+        logs-2:
+          target: [ c ]
+        logs-3:
+          target: [ c, e ]
+        logs-dual-query:
+          target: [ c, e ]
+        logs-4:
+          useCommonTable: true
+          target: [ c ]
+        logs-5:
+          useCommonTable: true
+          target: [ c ]
+        "*":
+          target: [ e ]
+  - name: IP
+    type: quesma-v1-processor-ingest
+    config:
+      indexes:
+        logs-1:
+          target: [ e ]
+        logs-2:
+          target: [ c ]
+        logs-3:
+          target: [ c, e ]
+        logs-dual-query:
+          target: [ c, e ]
+        logs-4:
+          useCommonTable: true
+          target: [ c ]
+        "*":
+          target: [ e ]
+        logs-5:
+          useCommonTable: true
+          target: [  ]
+
+pipelines:
+  - name: my-elasticsearch-proxy-read
+    frontendConnectors: [ elastic-query ]
+    processors: [ QP ]
+    backendConnectors: [ e, c ]
+  - name: my-elasticsearch-proxy-write
+    frontendConnectors: [ elastic-ingest ]
+    processors: [ IP ]
+    backendConnectors: [ e, c ]

ci/it/configs/quesma-with-two-pipelines.yml.template

@@ -0,0 +1,54 @@
+frontendConnectors:
+  - name: elastic-ingest
+    type: elasticsearch-fe-ingest
+    config:
+      listenPort: 8080
+  - name: elastic-query
+    type: elasticsearch-fe-query
+    config:
+      listenPort: 8080
+backendConnectors:
+  - name: my-minimal-elasticsearch
+    type: elasticsearch
+    config:
+      url: "http://{{ .elasticsearch_host }}:{{ .elasticsearch_port }}"
+      user: elastic
+      password: quesmaquesma
+  - name: my-hydrolix-instance
+    type: clickhouse-os
+    config:
+      url: clickhouse://{{ .clickhouse_host }}:{{ .clickhouse_port }}
+ingestStatistics: true
+processors:
+  - name: my-query-processor
+    type: quesma-v1-processor-query
+    config:
+      indexes:
+        siem:
+          target: [my-hydrolix-instance]
+        logs:
+          target: [my-hydrolix-instance]
+        test_index:
+          target: [my-minimal-elasticsearch]
+        "*":
+          target: [ my-minimal-elasticsearch ]
+  - name: my-ingest-processor
+    type: quesma-v1-processor-ingest
+    config:
+      indexes:
+        test_index:
+          target: [ my-minimal-elasticsearch ]
+        logs_disabled:
+          target: [  ]
+        "*":
+          target: [ my-minimal-elasticsearch ]
+pipelines:
+  - name: my-elasticsearch-proxy-read
+    frontendConnectors: [ elastic-query ]
+    processors: [ my-query-processor ]
+    backendConnectors: [ my-minimal-elasticsearch, my-hydrolix-instance ]
+  - name: my-elasticsearch-proxy-write
+    frontendConnectors: [ elastic-ingest ]
+    processors: [ my-ingest-processor ]
+    backendConnectors: [ my-minimal-elasticsearch, my-hydrolix-instance ]
+

ci/it/integration_test.go

@@ -26,3 +26,13 @@ func TestReadingClickHouseTablesIntegrationTestcase(t *testing.T) {
 	testCase := testcases.NewReadingClickHouseTablesIntegrationTestcase()
 	runIntegrationTest(t, testCase)
 }
+
+func TestQueryAndIngestPipelineTestcase(t *testing.T) {
+	testCase := testcases.NewQueryAndIngestPipelineTestcase()
+	runIntegrationTest(t, testCase)
+}
+
+func TestDualWriteAndCommonTableTestcase(t *testing.T) {
+	testCase := testcases.NewDualWriteAndCommonTableTestcase()
+	runIntegrationTest(t, testCase)
+}