feat: adding Elastic as a metrics provider (#3890)

SandeepD2697 · mowies · web-flow · commit 1802600c7aa7 · 2025-03-19T07:52:10.000+01:00
Signed-off-by: SandeepD2697 &lt;devinenisandeep7@gmail.com&gt;
Signed-off-by: Moritz Wiesinger &lt;moritz.wiesinger@dynatrace.com&gt;
Co-authored-by: Moritz Wiesinger &lt;moritz.wiesinger@dynatrace.com&gt;
diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -157,6 +157,8 @@ dupl
 dynamicinformer
 dynatrace
 ecr
+elastic
+elasticsearch
 elif
 ENABLEGITINFO
 endblock
@@ -392,6 +394,7 @@ loadtests
 LOCALBIN
 logf
 logr
+lte
 makefiles
 markdownfiles
 markdownlint
diff --git a/.github/scripts/.helm-tests/Openshift/result.yaml b/.github/scripts/.helm-tests/Openshift/result.yaml
@@ -13907,8 +13907,8 @@ spec:
                 type: string
               type:
                 description: Type represents the provider type. This can be one of
-                  cortex, datadog, dql, dynatrace, prometheus or thanos.
-                pattern: cortex|datadog|dql|dynatrace|prometheus|thanos
+                  cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
+                pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos
                 type: string
             required:
             - targetServer
diff --git a/.github/scripts/.helm-tests/default/result.yaml b/.github/scripts/.helm-tests/default/result.yaml
@@ -13907,8 +13907,8 @@ spec:
                 type: string
               type:
                 description: Type represents the provider type. This can be one of
-                  cortex, datadog, dql, dynatrace, prometheus or thanos.
-                pattern: cortex|datadog|dql|dynatrace|prometheus|thanos
+                  cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
+                pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos
                 type: string
             required:
             - targetServer
diff --git a/.github/scripts/.helm-tests/local-global-precedence/result.yaml b/.github/scripts/.helm-tests/local-global-precedence/result.yaml
@@ -14031,8 +14031,8 @@ spec:
                 type: string
               type:
                 description: Type represents the provider type. This can be one of
-                  cortex, datadog, dql, dynatrace, prometheus or thanos.
-                pattern: cortex|datadog|dql|dynatrace|prometheus|thanos
+                  cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
+                pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos
                 type: string
             required:
             - targetServer
diff --git a/.github/scripts/.helm-tests/metrics-only-with-apiservice-disabled/result.yaml b/.github/scripts/.helm-tests/metrics-only-with-apiservice-disabled/result.yaml
@@ -2593,8 +2593,8 @@ spec:
                 type: string
               type:
                 description: Type represents the provider type. This can be one of
-                  cortex, datadog, dql, dynatrace, prometheus or thanos.
-                pattern: cortex|datadog|dql|dynatrace|prometheus|thanos
+                  cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
+                pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos
                 type: string
             required:
             - targetServer
diff --git a/.github/scripts/.helm-tests/metrics-only/result.yaml b/.github/scripts/.helm-tests/metrics-only/result.yaml
@@ -2593,8 +2593,8 @@ spec:
                 type: string
               type:
                 description: Type represents the provider type. This can be one of
-                  cortex, datadog, dql, dynatrace, prometheus or thanos.
-                pattern: cortex|datadog|dql|dynatrace|prometheus|thanos
+                  cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
+                pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos
                 type: string
             required:
             - targetServer
diff --git a/.github/scripts/.helm-tests/metrics-with-certs/result.yaml b/.github/scripts/.helm-tests/metrics-with-certs/result.yaml
@@ -2608,8 +2608,8 @@ spec:
                 type: string
               type:
                 description: Type represents the provider type. This can be one of
-                  cortex, datadog, dql, dynatrace, prometheus or thanos.
-                pattern: cortex|datadog|dql|dynatrace|prometheus|thanos
+                  cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
+                pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos
                 type: string
             required:
             - targetServer
diff --git a/docs/docs/assets/crd/examples/yaml-synopsis.yaml b/docs/docs/assets/crd/examples/yaml-synopsis.yaml
@@ -4,7 +4,7 @@ metadata:
   name: <data-source-instance-name>
   namespace: <namespace>
 spec:
-  type: cortex | datadog | dql | dynatrace | prometheus | thanos
+  type: cortex | datadog | dql | dynatrace | prometheus | elastic | thanos
   targetServer: "<data-source-url>"
   secretKeyRef:
     name: <secret-name>
diff --git a/docs/docs/components/metrics-operator.md b/docs/docs/components/metrics-operator.md
@@ -13,7 +13,7 @@ of the application and infrastructure.
 
 While Kubernetes has ways to extend its metrics APIs, there are limitations,
 especially that they allow you to use only a single observability platform
-such as Prometheus, Thanos, Cortex, Dynatrace or Datadog.
+such as Prometheus, Thanos, Cortex, Dynatrace, Elastic or Datadog.
 The Keptn Metrics Operator solves this problem
 by providing a single entry point for
 all your metrics data, regardless of its source.
diff --git a/docs/docs/contribute/software/add-new-metric-provider.md b/docs/docs/contribute/software/add-new-metric-provider.md
@@ -95,7 +95,7 @@ The steps to create your own metrics provider are:
    [line](https://github.com/keptn/lifecycle-toolkit/blob/main/metrics-operator/api/v1/keptnmetricsprovider_types.go#L29)
    to look like this
 
-    `// +kubebuilder:validation:Pattern:=cortex|datadog|dql|dynatrace|prometheus|thanos|placeholder`.
+    `// +kubebuilder:validation:Pattern:=cortex|datadog|dql|dynatrace|prometheus|elastic|thanos|placeholder`.
 
      In the metric-operator directory run `make generate manifests` to update the metrics-operator crd config
      Then modify the helm chart and the helm chart crd validation to match the update in the metrics-operator crd config
diff --git a/docs/docs/core-concepts/index.md b/docs/docs/core-concepts/index.md
@@ -33,7 +33,7 @@ The Keptn metrics feature extends the functionality of
 
 * Handles observability data from multiple instances
   of multiple observability solutions
-  – Prometheus, Thanos, Cortex, Dynatrace, Datadog and others –
+  – Prometheus, Thanos, Cortex, Dynatrace, Datadog, Elastic and others –
   as well as data that comes directly from your cloud provider
   such as AWS, Google, or Azure.
 
diff --git a/docs/docs/getting-started/metrics.md b/docs/docs/getting-started/metrics.md
@@ -14,7 +14,7 @@ such as whether a rollout is good, or whether to scale up or down.
 
 Your observability data may come
 from multiple observability solutions --
-Prometheus, Thanos, Cortex, Dynatrace, Datadog and others --
+Prometheus, Thanos, Cortex, Dynatrace, Datadog, Elastic and others --
 or may be data that comes directly
 from your cloud provider such as AWS, Google, or Azure.
 The Keptn Metrics Server unifies and standardizes access to all this data.
diff --git a/docs/docs/installation/k8s.md b/docs/docs/installation/k8s.md
@@ -81,6 +81,7 @@ Your cluster should include the following:
   [Thanos](https://thanos.io/),
   [Cortex](https://cortexmetrics.io/),
   [Dynatrace](https://www.dynatrace.com/),
+  [Elastic](https://www.elastic.co/),
   or [Datadog](https://www.datadoghq.com/);
   you can use multiple instances of different data providers.
   These provide:
diff --git a/docs/docs/migrate/keptn/strategy.md b/docs/docs/migrate/keptn/strategy.md
@@ -269,7 +269,7 @@ Keptn v1
 [SLIs](https://v1.keptn.sh/docs/1.0.x/reference/files/sli/)
 (Service Level Indicators)
 represent queries from the data provider
-such as Prometheus, Thanos, Cortex, Dynatrace, or Datadog,
+such as Prometheus, Thanos, Cortex, Dynatrace, Elastic, or Datadog,
 which is configured as a Keptn integration.
 
 When migrating to Keptn, you need to define a
diff --git a/docs/docs/reference/api-reference/metrics/v1/index.md b/docs/docs/reference/api-reference/metrics/v1/index.md
@@ -373,7 +373,7 @@ _Appears in:_
 
 | Field | Description | Default | Optional |Validation |
 | --- | --- | --- | --- | --- |
-| `type` _string_ | Type represents the provider type. This can be one of cortex, datadog, dql, dynatrace, prometheus or thanos. || x | Optional: {} <br />Pattern: `cortex|datadog|dql|dynatrace|prometheus|thanos` <br /> |
+| `type` _string_ | Type represents the provider type. This can be one of cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. || x | Optional: {} <br />Pattern: `cortex|datadog|dql|dynatrace|prometheus|thanos|elastic` <br /> |
 | `targetServer` _string_ | TargetServer defines URL (including port and protocol) at which the metrics provider is reachable. || x |  |
 | `secretKeyRef` _[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#secretkeyselector-v1-core)_ | SecretKeyRef defines an optional secret for access credentials to the metrics provider. || ✓ | Optional: {} <br /> |
 | `insecureSkipTlsVerify` _boolean_ | InsecureSkipTlsVerify skips verification of the tls certificate when fetching metrics |false| ✓ |  |
diff --git a/docs/docs/reference/crd-reference/analysisvaluetemplate.md b/docs/docs/reference/crd-reference/analysisvaluetemplate.md
@@ -20,7 +20,7 @@ metadata:
   namespace: <namespace-where-this-resource-resides>
 spec:
   provider:
-    name: cortex | datadog | dql | dynatrace | prometheus | thanos
+    name: cortex | datadog | dql | dynatrace | prometheus | elastic | thanos
   query: <query>
 ```
 
diff --git a/docs/docs/reference/crd-reference/assets/keptnmetricsprovider-elastic.yaml b/docs/docs/reference/crd-reference/assets/keptnmetricsprovider-elastic.yaml
@@ -0,0 +1,19 @@
+apiVersion: metrics.keptn.sh/v1
+kind: KeptnMetricsProvider
+metadata:
+  name: elastic-provider
+  namespace: podtato-kubectl
+spec:
+  type: elastic
+  targetServer: "<elastic-url>"
+  secretKeyRef:
+    name: elastic-api-key
+    key: myCustomTokenKey
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elastic-api-key
+data:
+  myCustomTokenKey: my-token
+type: Opaque
diff --git a/docs/docs/reference/crd-reference/metricsprovider.md b/docs/docs/reference/crd-reference/metricsprovider.md
@@ -5,7 +5,7 @@ comments: true
 # KeptnMetricsProvider
 
 A `KeptnMetricsProvider` resource defines an instance of a data provider
-(such as Prometheus, Thanos, Cortex, Dynatrace, or Datadog)
+(such as Prometheus, Thanos, Cortex, Dynatrace, Elastic, or Datadog)
 that is used by one or more [KeptnMetric](metric.md) resources.
 
 One Keptn application can perform
@@ -117,6 +117,20 @@ For detailed information please look at the [Examples section](#examples).
     present in the linked Secret.
     Setting this field has no effect.
 
+=== "Elastic"
+
+    An example of Elastic as a metrics provider with a Secret holding
+    the authentication data looks like the following:
+ 
+    ```yaml
+    {% include "./assets/keptnmetricsprovider-elastic.yaml" %}
+    ```
+    > **Note**
+    When using Elastic as metrics provider you can
+    define the key name of your Elastic API Key stored in a secret,
+    which is not possible for Datadog, Prometheus, Cortex or Thanos.
+    For this example `myCustomTokenKey` was used. 
+
 === "Dynatrace and DQL"
 
     An example of Dynatrace as a metrics provider with a Secret holding
diff --git a/docs/docs/use-cases/non-k8s.md b/docs/docs/use-cases/non-k8s.md
@@ -179,13 +179,13 @@ similar to what the metrics evaluations of the
 Keptn v1 quality gates feature provided.
 The data used can come from multiple instances
 of multiple data providers
-(such as Prometheus, Thanos, Cortex, Dynatrace, and DataDog).
+(such as Prometheus, Thanos, Cortex, Dynatrace, Elastic, and DataDog).
 
 A Keptn analysis can be run for any application running anywhere
 as long Keptn can access a monitoring provider endpoint
 that serves metrics for the application.
 You can point to multiple instances of the supported monitoring providers
-(Prometheus, Thanos, Cortex, Dynatrace, Datadog, and dql)
+(Prometheus, Thanos, Cortex, Dynatrace, Datadog, Elastic, and dql)
 so the application itself can run anywhere.
 
 To implement a Keptn analysis for your deployment:
diff --git a/metrics-operator/api/v1/keptnmetricsprovider_types.go b/metrics-operator/api/v1/keptnmetricsprovider_types.go
@@ -26,8 +26,8 @@ import (
 // KeptnMetricsProviderSpec defines the desired state of KeptnMetricsProvider
 type KeptnMetricsProviderSpec struct {
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:validation:Pattern:=cortex|datadog|dql|dynatrace|prometheus|thanos
-	// Type represents the provider type. This can be one of cortex, datadog, dql, dynatrace, prometheus or thanos.
+	// +kubebuilder:validation:Pattern:=cortex|datadog|dql|dynatrace|prometheus|thanos|elastic
+	// Type represents the provider type. This can be one of cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
 	Type string `json:"type"`
 	// TargetServer defines URL (including port and protocol) at which the metrics provider is reachable.
 	TargetServer string `json:"targetServer"`
diff --git a/metrics-operator/api/v1/keptnmetricsprovider_types_test.go b/metrics-operator/api/v1/keptnmetricsprovider_types_test.go
@@ -57,6 +57,19 @@ func TestKeptnMetricsProvider_GetType(t *testing.T) {
 			},
 			want: "cortex",
 		},
+		{
+			name: "elastic provider type set",
+			fields: fields{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "provider1",
+				},
+				Spec: KeptnMetricsProviderSpec{
+					Type:         "elastic",
+					TargetServer: "",
+				},
+			},
+			want: "elastic",
+		},
 		{
 			name: "provider type not set, should return name",
 			fields: fields{
diff --git a/metrics-operator/chart/templates/keptnmetricsprovider-crd.yaml b/metrics-operator/chart/templates/keptnmetricsprovider-crd.yaml
@@ -81,8 +81,8 @@ spec:
                 type: string
               type:
                 description: Type represents the provider type. This can be one of
-                  cortex, datadog, dql, dynatrace, prometheus or thanos.
-                pattern: cortex|datadog|dql|dynatrace|prometheus|thanos
+                  cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
+                pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos
                 type: string
             required:
             - targetServer
diff --git a/metrics-operator/config/crd/bases/metrics.keptn.sh_keptnmetricsproviders.yaml b/metrics-operator/config/crd/bases/metrics.keptn.sh_keptnmetricsproviders.yaml
@@ -73,8 +73,8 @@ spec:
                 type: string
               type:
                 description: Type represents the provider type. This can be one of
-                  cortex, datadog, dql, dynatrace, prometheus or thanos.
-                pattern: cortex|datadog|dql|dynatrace|prometheus|thanos
+                  cortex, datadog, dql, dynatrace, prometheus, elastic or thanos.
+                pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos
                 type: string
             required:
             - targetServer
@@ -198,7 +198,7 @@ spec:
               type:
                 description: Type represents the provider type. This can be one of
                   prometheus, dynatrace, datadog, dql.
-                pattern: prometheus|dynatrace|datadog|dql
+                pattern: prometheus|dynatrace|datadog|dql|elastic
                 type: string
             required:
             - targetServer
@@ -264,7 +264,7 @@ spec:
               type:
                 description: Type represents the provider type. This can be one of
                   prometheus, dynatrace, datadog, dql.
-                pattern: prometheus|dynatrace|datadog|dql
+                pattern: prometheus|dynatrace|datadog|dql|elastic
                 type: string
             required:
             - targetServer
diff --git a/metrics-operator/controllers/analysis/provider_selector_test.go b/metrics-operator/controllers/analysis/provider_selector_test.go
@@ -246,7 +246,7 @@ func TestProvidersPool(t *testing.T) {
 
 func TestProvidersPool_StartProviders(t *testing.T) {
 
-	numJobs := 6
+	numJobs := 7
 	ctx, cancel := context.WithCancel(context.Background())
 	resChan := make(chan metricsapi.ProviderResult)
 	// Create a mock IObjectivesEvaluator, Client, and Logger for testing
@@ -273,7 +273,7 @@ func TestProvidersPool_StartProviders(t *testing.T) {
 	time.Sleep(time.Millisecond * 100)
 
 	// Assert the expected number of workers (goroutines) were started
-	require.Equal(t, 6, len(pool.providers))
+	require.Equal(t, 7, len(pool.providers))
 	require.Equal(t, numJobs, cap(pool.providers["prometheus"]))
 	// Stop the providers after testing
 	pool.StopProviders()
diff --git a/metrics-operator/controllers/common/providers/common.go b/metrics-operator/controllers/common/providers/common.go
@@ -6,6 +6,7 @@ const PrometheusProviderType = "prometheus"
 const ThanosProviderType = "thanos"
 const CortexProviderType = "cortex"
 const DataDogProviderType = "datadog"
+const ElasticProviderType = "elastic"
 
 var SupportedProviders = []string{
 	DynatraceProviderType,
@@ -14,4 +15,5 @@ var SupportedProviders = []string{
 	DataDogProviderType,
 	CortexProviderType,
 	ThanosProviderType,
+	ElasticProviderType,
 }
diff --git a/metrics-operator/controllers/common/providers/elastic/elastic.go b/metrics-operator/controllers/common/providers/elastic/elastic.go
diff --git a/metrics-operator/controllers/common/providers/elastic/elastic_test.go b/metrics-operator/controllers/common/providers/elastic/elastic_test.go
diff --git a/metrics-operator/controllers/common/providers/provider.go b/metrics-operator/controllers/common/providers/provider.go
diff --git a/metrics-operator/controllers/common/providers/provider_test.go b/metrics-operator/controllers/common/providers/provider_test.go
diff --git a/metrics-operator/go.mod b/metrics-operator/go.mod
diff --git a/metrics-operator/go.sum b/metrics-operator/go.sum
