Fix various elasticsearch usage issues

Signed-off-by: Steven Erb <stevenpc3@live.com>

Author: Stevenpc3

.github/workflows/lint-test.yaml

@@ -46,4 +46,4 @@ jobs:
 
         - uses: ./.github/actions/prepare-k8s
 
-        - run: make test-es
+        - run: make test-jaeger-connect-elasticsearch

Makefile

@@ -9,20 +9,39 @@ lint:
 test:
 	ct install --config ct.yaml
 
-.PHONY: test-es
-test-es:
+# This test clears the extraEnvs for elasticsearch provision and then runs the elasticsearch builtin tests
+# jaeger comes up in memory storage mode
+# .PHONY: test-es
+# test-es:
+# 	ct install --config ct.yaml \
+# 		--charts charts/jaeger \
+# 		--helm-extra-set-args " \
+# 		--set provisionDataStore.elasticsearch=true \
+# 		--set storage.type=elasticsearch \
+# 		--set elasticsearch.clusterHealthCheckParams=wait_for_status=yellow&timeout=10s \
+# 		--set elasticsearch.readinessProbe.initialDelaySeconds=60 \
+# 		--set elasticsearch.readinessProbe.failureThreshold=10 \
+# 		--set elasticsearch.volumeClaimTemplate.resources.requests.storage=5Gi \
+# 		--set elasticsearch.protocol=https \
+# 		--set elasticsearch.extraEnvs=null \
+# 		--set 'elasticsearch.extraEnvs[0].name=hello' \
+# 		--set 'elasticsearch.extraEnvs[0].value=world'"
+
+# This test will spin up the local provisioned elasticsearch and then connect jaeger to it not using memory store
+# it disables the built-in elasticsearch test since it just checks if it's up and jaeger connecting does the same
+# elasticsearch test also requires ssl and jaeger is configured to not require that for local connection
+.PHONY: test-jaeger-connect-elasticsearch
+test-jaeger-connect-elasticsearch:
 	ct install --config ct.yaml \
 		--charts charts/jaeger \
 		--helm-extra-set-args " \
 		--set provisionDataStore.elasticsearch=true \
 		--set storage.type=elasticsearch \
-		--set elasticsearch.master.masterOnly=false \
-		--set elasticsearch.master.replicaCount=1 \
-		--set elasticsearch.data.replicaCount=0 \
-		--set elasticsearch.coordinating.replicaCount=0 \
-		--set elasticsearch.ingest.replicaCount=0 \
-		--set elasticsearch.clusterHealthCheckParams=wait_for_status=yellow&timeout=10s \
+		--set elasticsearch.tests.enabled=false \
 		--set elasticsearch.readinessProbe.initialDelaySeconds=60 \
 		--set elasticsearch.readinessProbe.failureThreshold=10 \
-		--set storage.elasticsearch.scheme=https \
-		--set storage.elasticsearch.tls.insecure=true"
+		--set elasticsearch.volumeClaimTemplate.resources.requests.storage=5Gi \
+		--set elasticsearch.protocol=http \
+		--set config.extensions.jaeger_query.storage.traces=primary_store_elasticsearch \
+		--set config.extensions.jaeger_query.storage.traces_archive=archive_store_elasticsearch \
+		--set config.exporters.jaeger_storage_exporter.trace_storage=primary_store_elasticsearch "

README.md

@@ -33,7 +33,7 @@ To run the chart installation tests locally (simulating the CI environment), use
 
 ```bash
 make test      # default install suite
-make test-es   # Elasticsearch-provisioned suite
+make test-jaeger-connect-elasticsearch   # Elasticsearch-provisioned suite
 ```
 
 

charts/jaeger/Chart.yaml

@@ -3,7 +3,7 @@ appVersion: 2.14.1
 description: A Jaeger Helm chart for Kubernetes
 name: jaeger
 type: application
-version: 4.3.4
+version: 4.3.5
 # CronJobs require v1.21
 kubeVersion: ">= 1.21-0"
 keywords:

charts/jaeger/README.md

@@ -50,6 +50,10 @@ helm install jaeger jaegertracing/jaeger
 
 ## Configuration
 
+There are several examples of how to configure you can reference at:
+
+https://github.com/jaegertracing/jaeger/tree/main/cmd/jaeger
+
 ### 1. In-Memory (Default)
 Ideal for testing. No persistence.
 ```bash
@@ -59,31 +63,71 @@ helm install jaeger jaegertracing/jaeger
 ### 2. Elasticsearch (Production Recommended)
 Configure Jaeger to connect to Elasticsearch using the native config syntax.
 
-**values.yaml Example:**
+You can either use the internal provisioned elasticsearch or point to an external elasticsearch.
+
+The internal elasticsearch is configured to __NOT__ require a password or username by default.
+
+#### Internal Provisioned Elasticsearch example:
+
+**values.yaml**
 ```yaml
 # Use the provisioned Elasticsearch subchart
 provisionDataStore:
   elasticsearch: true
 
-# Or connect to an external Elasticsearch cluster by customizing the config:
 config:
   extensions:
+    jaeger_query:
+      storage:
+        traces: primary_store_elasticsearch
+        traces_archive: archive_store_elasticsearch
+  exporters:
+    jaeger_storage_exporter:
+      trace_storage: primary_store_elasticsearch
+```
+
+#### External Elasticsearch example:
+
+**values.yaml**
+```yaml
+# External is very similar but you need to configure the urls
+config:
+  extensions:
+    jaeger_query:
+      storage:
+        traces: primary_store_elasticsearch
+        traces_archive: archive_store_elasticsearch
     jaeger_storage:
       backends:
-        primary_store:
+        primary_store_elasticsearch:
+          elasticsearch:
+            server_urls: ["http://my-elasticsearch-master:9200"]
+            auth:
+              basic:
+                username: elastic
+                password: changeme
+        archive_store_elasticsearch:
           elasticsearch:
-            server_urls: ["http://elasticsearch:9200"]
-            username: elastic
-            password: changeme
+            server_urls: ["http://my-elasticsearch-master:9200"]
+            auth:
+              basic:
+                username: elastic
+                password: changeme
+  exporters:
+    jaeger_storage_exporter:
+      trace_storage: primary_store_elasticsearch
 ```
 
 **Running Maintenance Jobs:**
-To run Index Cleaner or Rollover jobs, enable them. They auto-configure when `provisionDataStore.elasticsearch` is enabled:
+To run Index Cleaner or Rollover jobs, enable them. They require configuration if using external by setting ```storage.type=elasticsearch``` and adjusting ```storage.elasticsearch``` values.  See that section in values.yaml for more details.
+
 ```yaml
 esIndexCleaner:
   enabled: true
 ```
 
+* es-rollover __requires__ elasticsearch to be running __BEFORE__ install so the hook job can run.
+
 ### 3. Cassandra
 This chart does not provision a Cassandra cluster. To use Cassandra storage, you must provide your own Cassandra instance and configure Jaeger via the native config syntax:
 
@@ -110,14 +154,16 @@ config:
 
 ### 4. Spark Dependencies
 
-To run the Spark dependencies job (for dependency links graph):
+To run the Spark dependencies job (for dependency links graph) set ```spark.enabled=true```
+
+Below is an example of how to set overrides.  Please see the values.yaml for more examples:
+
+The values are populated under ```storage.elasticsearch``` Please see comments in values.yaml for more details.
 
 ```yaml
 spark:
   enabled: true
   extraEnv:
-    - name: ES_NODES
-      value: http://elasticsearch:9200
     - name: ES_NODES_WAN_ONLY
       value: "true"
 ```

charts/jaeger/templates/_helpers.tpl

@@ -233,24 +233,21 @@ Elasticsearch related environment variables
 {{- define "elasticsearch.env" -}}
 {{- if or .Values.provisionDataStore.elasticsearch (eq .Values.storage.type "elasticsearch") -}}
 {{- $es := .Values.storage.elasticsearch | default dict -}}
-{{- $scheme := $es.scheme | default "http" -}}
-{{- $port := $es.port | default 9200 -}}
 {{- $user := $es.user | default "elastic" -}}
-{{- $password := .Values.elasticsearch.secret.password | default "changeme" -}}
+{{- $password := $es.password | default "changeme" -}}
+{{- $url := $es.url | default "http://elasticsearch-master:9200" -}}
 - name: ES_SERVER_URLS
-  value: "{{ $scheme }}://elasticsearch-master:{{ $port }}"
+  value: {{ $url | quote }}
+- name: ES_NODES
+  value: {{ $url | quote }}
 - name: ES_USERNAME
   value: {{ $user | quote }}
 - name: ES_PASSWORD
   value: {{ $password | quote }}
 {{- /* Handle TLS insecurity */ -}}
-{{- if or ( and $es.tls ( $es.tls.insecure ) ) ( eq $scheme "https" ) }}
-  {{- if $es.tls }}
-    {{- if $es.tls.insecure }}
+{{- if and (($es).tls).enabled (($es).tls).insecure }}
 - name: ES_TLS_SKIP_HOST_VERIFY
   value: "true"
-    {{- end }}
-  {{- end }}
 {{- end }}
 {{- end }}
 {{- end -}}