diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index a315d7251f..b5b26d0592 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -157,6 +157,8 @@ dupl dynamicinformer dynatrace ecr +elastic +elasticsearch elif ENABLEGITINFO endblock @@ -392,6 +394,7 @@ loadtests LOCALBIN logf logr +lte makefiles markdownfiles markdownlint diff --git a/.github/scripts/.helm-tests/Openshift/result.yaml b/.github/scripts/.helm-tests/Openshift/result.yaml index b425aa3c26..145b14e5d9 100644 --- a/.github/scripts/.helm-tests/Openshift/result.yaml +++ b/.github/scripts/.helm-tests/Openshift/result.yaml @@ -13907,8 +13907,8 @@ spec: type: string type: description: Type represents the provider type. This can be one of - cortex, datadog, dql, dynatrace, prometheus or thanos. - pattern: cortex|datadog|dql|dynatrace|prometheus|thanos + cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. + pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos type: string required: - targetServer diff --git a/.github/scripts/.helm-tests/default/result.yaml b/.github/scripts/.helm-tests/default/result.yaml index 54d7be8b25..91e8f83f0a 100644 --- a/.github/scripts/.helm-tests/default/result.yaml +++ b/.github/scripts/.helm-tests/default/result.yaml @@ -13907,8 +13907,8 @@ spec: type: string type: description: Type represents the provider type. This can be one of - cortex, datadog, dql, dynatrace, prometheus or thanos. - pattern: cortex|datadog|dql|dynatrace|prometheus|thanos + cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. + pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos type: string required: - targetServer diff --git a/.github/scripts/.helm-tests/local-global-precedence/result.yaml b/.github/scripts/.helm-tests/local-global-precedence/result.yaml index 9c9cbcb527..f3b3e1d0b7 100644 --- a/.github/scripts/.helm-tests/local-global-precedence/result.yaml +++ b/.github/scripts/.helm-tests/local-global-precedence/result.yaml @@ -14031,8 +14031,8 @@ spec: type: string type: description: Type represents the provider type. This can be one of - cortex, datadog, dql, dynatrace, prometheus or thanos. - pattern: cortex|datadog|dql|dynatrace|prometheus|thanos + cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. + pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos type: string required: - targetServer diff --git a/.github/scripts/.helm-tests/metrics-only-with-apiservice-disabled/result.yaml b/.github/scripts/.helm-tests/metrics-only-with-apiservice-disabled/result.yaml index 75bcd3e380..0381eaba98 100644 --- a/.github/scripts/.helm-tests/metrics-only-with-apiservice-disabled/result.yaml +++ b/.github/scripts/.helm-tests/metrics-only-with-apiservice-disabled/result.yaml @@ -2593,8 +2593,8 @@ spec: type: string type: description: Type represents the provider type. This can be one of - cortex, datadog, dql, dynatrace, prometheus or thanos. - pattern: cortex|datadog|dql|dynatrace|prometheus|thanos + cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. + pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos type: string required: - targetServer diff --git a/.github/scripts/.helm-tests/metrics-only/result.yaml b/.github/scripts/.helm-tests/metrics-only/result.yaml index 067a8696e9..4a82d9fffd 100644 --- a/.github/scripts/.helm-tests/metrics-only/result.yaml +++ b/.github/scripts/.helm-tests/metrics-only/result.yaml @@ -2593,8 +2593,8 @@ spec: type: string type: description: Type represents the provider type. This can be one of - cortex, datadog, dql, dynatrace, prometheus or thanos. - pattern: cortex|datadog|dql|dynatrace|prometheus|thanos + cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. + pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos type: string required: - targetServer diff --git a/.github/scripts/.helm-tests/metrics-with-certs/result.yaml b/.github/scripts/.helm-tests/metrics-with-certs/result.yaml index e61693e0d7..03ef0a386c 100644 --- a/.github/scripts/.helm-tests/metrics-with-certs/result.yaml +++ b/.github/scripts/.helm-tests/metrics-with-certs/result.yaml @@ -2608,8 +2608,8 @@ spec: type: string type: description: Type represents the provider type. This can be one of - cortex, datadog, dql, dynatrace, prometheus or thanos. - pattern: cortex|datadog|dql|dynatrace|prometheus|thanos + cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. + pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos type: string required: - targetServer diff --git a/docs/docs/assets/crd/examples/yaml-synopsis.yaml b/docs/docs/assets/crd/examples/yaml-synopsis.yaml index 96d659af15..90f59e36fe 100644 --- a/docs/docs/assets/crd/examples/yaml-synopsis.yaml +++ b/docs/docs/assets/crd/examples/yaml-synopsis.yaml @@ -4,7 +4,7 @@ metadata: name: namespace: spec: - type: cortex | datadog | dql | dynatrace | prometheus | thanos + type: cortex | datadog | dql | dynatrace | prometheus | elastic | thanos targetServer: "" secretKeyRef: name: diff --git a/docs/docs/components/metrics-operator.md b/docs/docs/components/metrics-operator.md index 2f37cb5424..3ceabc9ede 100644 --- a/docs/docs/components/metrics-operator.md +++ b/docs/docs/components/metrics-operator.md @@ -13,7 +13,7 @@ of the application and infrastructure. While Kubernetes has ways to extend its metrics APIs, there are limitations, especially that they allow you to use only a single observability platform -such as Prometheus, Thanos, Cortex, Dynatrace or Datadog. +such as Prometheus, Thanos, Cortex, Dynatrace, Elastic or Datadog. The Keptn Metrics Operator solves this problem by providing a single entry point for all your metrics data, regardless of its source. diff --git a/docs/docs/contribute/software/add-new-metric-provider.md b/docs/docs/contribute/software/add-new-metric-provider.md index 14a1732eb2..0fd5531d59 100644 --- a/docs/docs/contribute/software/add-new-metric-provider.md +++ b/docs/docs/contribute/software/add-new-metric-provider.md @@ -95,7 +95,7 @@ The steps to create your own metrics provider are: [line](https://github.com/keptn/lifecycle-toolkit/blob/main/metrics-operator/api/v1/keptnmetricsprovider_types.go#L29) to look like this - `// +kubebuilder:validation:Pattern:=cortex|datadog|dql|dynatrace|prometheus|thanos|placeholder`. + `// +kubebuilder:validation:Pattern:=cortex|datadog|dql|dynatrace|prometheus|elastic|thanos|placeholder`. In the metric-operator directory run `make generate manifests` to update the metrics-operator crd config Then modify the helm chart and the helm chart crd validation to match the update in the metrics-operator crd config diff --git a/docs/docs/core-concepts/index.md b/docs/docs/core-concepts/index.md index 88598917ef..b3ee4d963c 100644 --- a/docs/docs/core-concepts/index.md +++ b/docs/docs/core-concepts/index.md @@ -33,7 +33,7 @@ The Keptn metrics feature extends the functionality of * Handles observability data from multiple instances of multiple observability solutions - – Prometheus, Thanos, Cortex, Dynatrace, Datadog and others – + – Prometheus, Thanos, Cortex, Dynatrace, Datadog, Elastic and others – as well as data that comes directly from your cloud provider such as AWS, Google, or Azure. diff --git a/docs/docs/getting-started/metrics.md b/docs/docs/getting-started/metrics.md index c0a52eac47..110baec9ca 100644 --- a/docs/docs/getting-started/metrics.md +++ b/docs/docs/getting-started/metrics.md @@ -14,7 +14,7 @@ such as whether a rollout is good, or whether to scale up or down. Your observability data may come from multiple observability solutions -- -Prometheus, Thanos, Cortex, Dynatrace, Datadog and others -- +Prometheus, Thanos, Cortex, Dynatrace, Datadog, Elastic and others -- or may be data that comes directly from your cloud provider such as AWS, Google, or Azure. The Keptn Metrics Server unifies and standardizes access to all this data. diff --git a/docs/docs/installation/k8s.md b/docs/docs/installation/k8s.md index 5168b251b4..1931d94660 100644 --- a/docs/docs/installation/k8s.md +++ b/docs/docs/installation/k8s.md @@ -81,6 +81,7 @@ Your cluster should include the following: [Thanos](https://thanos.io/), [Cortex](https://cortexmetrics.io/), [Dynatrace](https://www.dynatrace.com/), + [Elastic](https://www.elastic.co/), or [Datadog](https://www.datadoghq.com/); you can use multiple instances of different data providers. These provide: diff --git a/docs/docs/migrate/keptn/strategy.md b/docs/docs/migrate/keptn/strategy.md index 5ed58cb985..cc9f0e0ae0 100644 --- a/docs/docs/migrate/keptn/strategy.md +++ b/docs/docs/migrate/keptn/strategy.md @@ -269,7 +269,7 @@ Keptn v1 [SLIs](https://v1.keptn.sh/docs/1.0.x/reference/files/sli/) (Service Level Indicators) represent queries from the data provider -such as Prometheus, Thanos, Cortex, Dynatrace, or Datadog, +such as Prometheus, Thanos, Cortex, Dynatrace, Elastic, or Datadog, which is configured as a Keptn integration. When migrating to Keptn, you need to define a diff --git a/docs/docs/reference/api-reference/metrics/v1/index.md b/docs/docs/reference/api-reference/metrics/v1/index.md index 049c78b7db..a720033ca8 100644 --- a/docs/docs/reference/api-reference/metrics/v1/index.md +++ b/docs/docs/reference/api-reference/metrics/v1/index.md @@ -373,7 +373,7 @@ _Appears in:_ | Field | Description | Default | Optional |Validation | | --- | --- | --- | --- | --- | -| `type` _string_ | Type represents the provider type. This can be one of cortex, datadog, dql, dynatrace, prometheus or thanos. || x | Optional: {}
Pattern: `cortex|datadog|dql|dynatrace|prometheus|thanos`
| +| `type` _string_ | Type represents the provider type. This can be one of cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. || x | Optional: {}
Pattern: `cortex|datadog|dql|dynatrace|prometheus|thanos|elastic`
| | `targetServer` _string_ | TargetServer defines URL (including port and protocol) at which the metrics provider is reachable. || x | | | `secretKeyRef` _[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#secretkeyselector-v1-core)_ | SecretKeyRef defines an optional secret for access credentials to the metrics provider. || ✓ | Optional: {}
| | `insecureSkipTlsVerify` _boolean_ | InsecureSkipTlsVerify skips verification of the tls certificate when fetching metrics |false| ✓ | | diff --git a/docs/docs/reference/crd-reference/analysisvaluetemplate.md b/docs/docs/reference/crd-reference/analysisvaluetemplate.md index ffeee309ae..472baac427 100644 --- a/docs/docs/reference/crd-reference/analysisvaluetemplate.md +++ b/docs/docs/reference/crd-reference/analysisvaluetemplate.md @@ -20,7 +20,7 @@ metadata: namespace: spec: provider: - name: cortex | datadog | dql | dynatrace | prometheus | thanos + name: cortex | datadog | dql | dynatrace | prometheus | elastic | thanos query: ``` diff --git a/docs/docs/reference/crd-reference/assets/keptnmetricsprovider-elastic.yaml b/docs/docs/reference/crd-reference/assets/keptnmetricsprovider-elastic.yaml new file mode 100644 index 0000000000..efaaa5c306 --- /dev/null +++ b/docs/docs/reference/crd-reference/assets/keptnmetricsprovider-elastic.yaml @@ -0,0 +1,19 @@ +apiVersion: metrics.keptn.sh/v1 +kind: KeptnMetricsProvider +metadata: + name: elastic-provider + namespace: podtato-kubectl +spec: + type: elastic + targetServer: "" + secretKeyRef: + name: elastic-api-key + key: myCustomTokenKey +--- +apiVersion: v1 +kind: Secret +metadata: + name: elastic-api-key +data: + myCustomTokenKey: my-token +type: Opaque diff --git a/docs/docs/reference/crd-reference/metricsprovider.md b/docs/docs/reference/crd-reference/metricsprovider.md index d61fb52874..303d71acb1 100644 --- a/docs/docs/reference/crd-reference/metricsprovider.md +++ b/docs/docs/reference/crd-reference/metricsprovider.md @@ -5,7 +5,7 @@ comments: true # KeptnMetricsProvider A `KeptnMetricsProvider` resource defines an instance of a data provider -(such as Prometheus, Thanos, Cortex, Dynatrace, or Datadog) +(such as Prometheus, Thanos, Cortex, Dynatrace, Elastic, or Datadog) that is used by one or more [KeptnMetric](metric.md) resources. One Keptn application can perform @@ -117,6 +117,20 @@ For detailed information please look at the [Examples section](#examples). present in the linked Secret. Setting this field has no effect. +=== "Elastic" + + An example of Elastic as a metrics provider with a Secret holding + the authentication data looks like the following: + + ```yaml + {% include "./assets/keptnmetricsprovider-elastic.yaml" %} + ``` + > **Note** + When using Elastic as metrics provider you can + define the key name of your Elastic API Key stored in a secret, + which is not possible for Datadog, Prometheus, Cortex or Thanos. + For this example `myCustomTokenKey` was used. + === "Dynatrace and DQL" An example of Dynatrace as a metrics provider with a Secret holding diff --git a/docs/docs/use-cases/non-k8s.md b/docs/docs/use-cases/non-k8s.md index cddcfd40be..54b4c98b58 100644 --- a/docs/docs/use-cases/non-k8s.md +++ b/docs/docs/use-cases/non-k8s.md @@ -179,13 +179,13 @@ similar to what the metrics evaluations of the Keptn v1 quality gates feature provided. The data used can come from multiple instances of multiple data providers -(such as Prometheus, Thanos, Cortex, Dynatrace, and DataDog). +(such as Prometheus, Thanos, Cortex, Dynatrace, Elastic, and DataDog). A Keptn analysis can be run for any application running anywhere as long Keptn can access a monitoring provider endpoint that serves metrics for the application. You can point to multiple instances of the supported monitoring providers -(Prometheus, Thanos, Cortex, Dynatrace, Datadog, and dql) +(Prometheus, Thanos, Cortex, Dynatrace, Datadog, Elastic, and dql) so the application itself can run anywhere. To implement a Keptn analysis for your deployment: diff --git a/metrics-operator/api/v1/keptnmetricsprovider_types.go b/metrics-operator/api/v1/keptnmetricsprovider_types.go index 48fbaf08b0..9d4f65e9cb 100644 --- a/metrics-operator/api/v1/keptnmetricsprovider_types.go +++ b/metrics-operator/api/v1/keptnmetricsprovider_types.go @@ -26,8 +26,8 @@ import ( // KeptnMetricsProviderSpec defines the desired state of KeptnMetricsProvider type KeptnMetricsProviderSpec struct { // +kubebuilder:validation:Optional - // +kubebuilder:validation:Pattern:=cortex|datadog|dql|dynatrace|prometheus|thanos - // Type represents the provider type. This can be one of cortex, datadog, dql, dynatrace, prometheus or thanos. + // +kubebuilder:validation:Pattern:=cortex|datadog|dql|dynatrace|prometheus|thanos|elastic + // Type represents the provider type. This can be one of cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. Type string `json:"type"` // TargetServer defines URL (including port and protocol) at which the metrics provider is reachable. TargetServer string `json:"targetServer"` diff --git a/metrics-operator/api/v1/keptnmetricsprovider_types_test.go b/metrics-operator/api/v1/keptnmetricsprovider_types_test.go index f7e8909246..568683093c 100644 --- a/metrics-operator/api/v1/keptnmetricsprovider_types_test.go +++ b/metrics-operator/api/v1/keptnmetricsprovider_types_test.go @@ -57,6 +57,19 @@ func TestKeptnMetricsProvider_GetType(t *testing.T) { }, want: "cortex", }, + { + name: "elastic provider type set", + fields: fields{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider1", + }, + Spec: KeptnMetricsProviderSpec{ + Type: "elastic", + TargetServer: "", + }, + }, + want: "elastic", + }, { name: "provider type not set, should return name", fields: fields{ diff --git a/metrics-operator/chart/templates/keptnmetricsprovider-crd.yaml b/metrics-operator/chart/templates/keptnmetricsprovider-crd.yaml index 50a24fa6d5..9e4009e4ba 100644 --- a/metrics-operator/chart/templates/keptnmetricsprovider-crd.yaml +++ b/metrics-operator/chart/templates/keptnmetricsprovider-crd.yaml @@ -81,8 +81,8 @@ spec: type: string type: description: Type represents the provider type. This can be one of - cortex, datadog, dql, dynatrace, prometheus or thanos. - pattern: cortex|datadog|dql|dynatrace|prometheus|thanos + cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. + pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos type: string required: - targetServer diff --git a/metrics-operator/config/crd/bases/metrics.keptn.sh_keptnmetricsproviders.yaml b/metrics-operator/config/crd/bases/metrics.keptn.sh_keptnmetricsproviders.yaml index ec61b83506..9248a12eaf 100644 --- a/metrics-operator/config/crd/bases/metrics.keptn.sh_keptnmetricsproviders.yaml +++ b/metrics-operator/config/crd/bases/metrics.keptn.sh_keptnmetricsproviders.yaml @@ -73,8 +73,8 @@ spec: type: string type: description: Type represents the provider type. This can be one of - cortex, datadog, dql, dynatrace, prometheus or thanos. - pattern: cortex|datadog|dql|dynatrace|prometheus|thanos + cortex, datadog, dql, dynatrace, prometheus, elastic or thanos. + pattern: cortex|datadog|dql|dynatrace|prometheus|elastic|thanos type: string required: - targetServer @@ -198,7 +198,7 @@ spec: type: description: Type represents the provider type. This can be one of prometheus, dynatrace, datadog, dql. - pattern: prometheus|dynatrace|datadog|dql + pattern: prometheus|dynatrace|datadog|dql|elastic type: string required: - targetServer @@ -264,7 +264,7 @@ spec: type: description: Type represents the provider type. This can be one of prometheus, dynatrace, datadog, dql. - pattern: prometheus|dynatrace|datadog|dql + pattern: prometheus|dynatrace|datadog|dql|elastic type: string required: - targetServer diff --git a/metrics-operator/controllers/analysis/provider_selector_test.go b/metrics-operator/controllers/analysis/provider_selector_test.go index a067bab2d0..d3260c13d4 100644 --- a/metrics-operator/controllers/analysis/provider_selector_test.go +++ b/metrics-operator/controllers/analysis/provider_selector_test.go @@ -246,7 +246,7 @@ func TestProvidersPool(t *testing.T) { func TestProvidersPool_StartProviders(t *testing.T) { - numJobs := 6 + numJobs := 7 ctx, cancel := context.WithCancel(context.Background()) resChan := make(chan metricsapi.ProviderResult) // Create a mock IObjectivesEvaluator, Client, and Logger for testing @@ -273,7 +273,7 @@ func TestProvidersPool_StartProviders(t *testing.T) { time.Sleep(time.Millisecond * 100) // Assert the expected number of workers (goroutines) were started - require.Equal(t, 6, len(pool.providers)) + require.Equal(t, 7, len(pool.providers)) require.Equal(t, numJobs, cap(pool.providers["prometheus"])) // Stop the providers after testing pool.StopProviders() diff --git a/metrics-operator/controllers/common/providers/common.go b/metrics-operator/controllers/common/providers/common.go index d59e381c12..60db3fc775 100644 --- a/metrics-operator/controllers/common/providers/common.go +++ b/metrics-operator/controllers/common/providers/common.go @@ -6,6 +6,7 @@ const PrometheusProviderType = "prometheus" const ThanosProviderType = "thanos" const CortexProviderType = "cortex" const DataDogProviderType = "datadog" +const ElasticProviderType = "elastic" var SupportedProviders = []string{ DynatraceProviderType, @@ -14,4 +15,5 @@ var SupportedProviders = []string{ DataDogProviderType, CortexProviderType, ThanosProviderType, + ElasticProviderType, } diff --git a/metrics-operator/controllers/common/providers/elastic/elastic.go b/metrics-operator/controllers/common/providers/elastic/elastic.go new file mode 100644 index 0000000000..bbdd8416d7 --- /dev/null +++ b/metrics-operator/controllers/common/providers/elastic/elastic.go @@ -0,0 +1,178 @@ +package elastic + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + elastic "github.com/elastic/go-elasticsearch/v8" + "github.com/go-logr/logr" + metricsapi "github.com/keptn/lifecycle-toolkit/metrics-operator/api/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + warningLogStringElastic = "%s API returned warnings: %s" +) + +type KeptnElasticProvider struct { + Log logr.Logger + K8sClient client.Client + Elastic *elastic.Client +} + +// GetElasticClient will create a new elastic client +func GetElasticClient(provider metricsapi.KeptnMetricsProvider) (*elastic.Client, error) { + es, err := elastic.NewClient(elastic.Config{ + Addresses: []string{provider.Spec.TargetServer}, + APIKey: provider.Spec.SecretKeyRef.Key, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: provider.Spec.InsecureSkipTlsVerify, + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to create Elasticsearch client: %w", err) + } + return es, nil +} + +// FetchAnalysisValue will fetch analysis value depends on query and the metrics provided as input +func (r *KeptnElasticProvider) FetchAnalysisValue(ctx context.Context, query string, analysis metricsapi.Analysis, provider *metricsapi.KeptnMetricsProvider) (string, error) { + // Retrieve the AnalysisDefinition referenced in Analysis + var analysisDef metricsapi.AnalysisDefinition + err := r.K8sClient.Get(ctx, client.ObjectKey{ + Name: analysis.Spec.AnalysisDefinition.Name, + Namespace: analysis.Namespace, + }, &analysisDef) + + if err != nil { + r.Log.Error(err, "Failed to retrieve AnalysisDefinition") + return "", fmt.Errorf("failed to get AnalysisDefinition: %w", err) + } + + // Extract the referenced AnalysisValueTemplate name + if len(analysisDef.Spec.Objectives) == 0 { + return "", fmt.Errorf("no objectives defined in AnalysisDefinition") + } + + templateName := analysisDef.Spec.Objectives[0].AnalysisValueTemplateRef.Name + r.Log.Info("Found referenced AnalysisValueTemplate", "templateName", templateName) + // Retrieve the AnalysisValueTemplate using the extracted name + var template metricsapi.AnalysisValueTemplate + err = r.K8sClient.Get(ctx, client.ObjectKey{ + Name: templateName, + Namespace: analysis.Namespace, + }, &template) + + if err != nil { + r.Log.Error(err, "Failed to retrieve AnalysisValueTemplate") + return "", fmt.Errorf("failed to get AnalysisValueTemplate: %w", err) + } + + // Extract metricPath from args + metricPathStr, exists := analysis.Spec.Args["metricPath"] + if !exists || metricPathStr == "" { + return "", fmt.Errorf("metric path is missing in AnalysisValueTemplate annotations") + } + + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + + result, err := r.runElasticQuery(ctx, query) + if err != nil { + return "", err + } + + r.Log.Info("Elasticsearch query result", "result", result) + return r.extractMetric(result, metricPathStr) +} + +// EvaluateQuery takes query as a input but doesn't return anything +func (r *KeptnElasticProvider) EvaluateQuery(ctx context.Context, metric metricsapi.KeptnMetric, provider metricsapi.KeptnMetricsProvider) (string, []byte, error) { + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + + result, err := r.runElasticQuery(ctx, metric.Spec.Query) + if err != nil { + return "", nil, err + } + + metricValue, err := r.extractMetric(result, "") + if err != nil { + return "", nil, err + } + + return metricValue, []byte{}, nil +} + +func (r *KeptnElasticProvider) EvaluateQueryForStep(ctx context.Context, metric metricsapi.KeptnMetric, provider metricsapi.KeptnMetricsProvider) ([]string, []byte, error) { + r.Log.Info("EvaluateQueryForStep called but not implemented") + return nil, nil, nil +} + +// runElasticQuery runs query on elastic search to get output from elasticsearch +func (r *KeptnElasticProvider) runElasticQuery(ctx context.Context, query string) (map[string]interface{}, error) { + + res, err := r.Elastic.Search( + r.Elastic.Search.WithContext(ctx), + r.Elastic.Search.WithBody(strings.NewReader(query)), + ) + if err != nil { + return nil, fmt.Errorf("failed to execute Elasticsearch query: %w", err) + } + defer res.Body.Close() + + if warnings, ok := res.Header["Warning"]; ok { + r.Log.Info(fmt.Sprintf(warningLogStringElastic, "Elasticsearch", warnings)) + } + + var result map[string]interface{} + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to parse Elasticsearch response: %w", err) + } + + return result, nil +} + +// extractMetric will parse the result and return the metrics which we input to the function +func (r *KeptnElasticProvider) extractMetric(result map[string]interface{}, metricPathStr string) (string, error) { + convertedResult := convertResultTOMap(result) + for k, v := range convertedResult { + if strings.Contains(k, metricPathStr) { + return fmt.Sprintf("%f", v), nil + } + } + return "", nil +} + +// convertResultTOMap recursively converts map[string]interface{} to map[string]float64 +func convertResultTOMap(input map[string]interface{}) map[string]float64 { + output := make(map[string]float64) + for key, value := range input { + switch v := value.(type) { + case float64: + output[key] = v + case float32: + output[key] = float64(v) + case int: + output[key] = float64(v) + case int32: + output[key] = float64(v) + case int64: + output[key] = float64(v) + case map[string]interface{}: + nestedMap := convertResultTOMap(v) + for nestedKey, nestedValue := range nestedMap { + output[key+"."+nestedKey] = nestedValue + } + default: + } + } + return output +} diff --git a/metrics-operator/controllers/common/providers/elastic/elastic_test.go b/metrics-operator/controllers/common/providers/elastic/elastic_test.go new file mode 100644 index 0000000000..436597129c --- /dev/null +++ b/metrics-operator/controllers/common/providers/elastic/elastic_test.go @@ -0,0 +1,252 @@ +package elastic + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/go-logr/logr" + metricsapi "github.com/keptn/lifecycle-toolkit/metrics-operator/api/v1" + "github.com/stretchr/testify/assert" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestGetElasticClient(t *testing.T) { + tests := []struct { + name string + expectedError bool + }{ + { + name: "Success - Elasticsearch client created", + expectedError: false, + }, + { + name: "Failure - Invalid connection", + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + provider := metricsapi.KeptnMetricsProvider{} + client, err := GetElasticClient(provider) + + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.NotNil(t, client) + assert.IsType(t, &elasticsearch.Client{}, client) + } + }) + } +} + +func TestFetchAnalysisValue(t *testing.T) { + provider := &KeptnElasticProvider{ + K8sClient: fake.NewFakeClient(), + Log: logr.Logger{}, + } + + tests := []struct { + name string + query string + analysis metricsapi.Analysis + expectedError bool + }{ + { + name: "Failure - Missing AnalysisDefinition", + query: "SELECT avg(cpu) FROM metrics", + analysis: metricsapi.Analysis{ + Spec: metricsapi.AnalysisSpec{ + Args: map[string]string{"metricPath": "metrics.cpu"}, + AnalysisDefinition: metricsapi.ObjectReference{ + Name: "non-existent-definition", + }, + }, + }, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + value, err := provider.FetchAnalysisValue(ctx, tt.query, tt.analysis, &metricsapi.KeptnMetricsProvider{}) + + if tt.expectedError { + assert.Error(t, err) + assert.Empty(t, value) + } else { + assert.NoError(t, err) + assert.NotEmpty(t, value) + } + }) + } +} + +func TestEvaluateQueryForStep(t *testing.T) { + provider := &KeptnElasticProvider{ + Log: logr.Discard(), + } + + tests := []struct { + name string + metric metricsapi.KeptnMetric + wantErr bool + }{ + { + name: "Unimplemented function returns nil values", + metric: metricsapi.KeptnMetric{}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + result, rawData, err := provider.EvaluateQueryForStep(ctx, tt.metric, metricsapi.KeptnMetricsProvider{}) + + assert.Nil(t, result) + assert.Nil(t, rawData) + assert.NoError(t, err) + }) + } +} + +func TestRunElasticQuery(t *testing.T) { + esClient, err := elasticsearch.NewDefaultClient() + if err != nil { + t.Fatalf("Failed to create Elasticsearch client: %v", err) + } + + provider := &KeptnElasticProvider{ + Elastic: esClient, + Log: logr.Discard(), + } + + tests := []struct { + name string + query string + expectedError bool + }{ + { + name: "Success - Valid Query", + query: `{"query": {"match_all": {}}}`, + expectedError: true, + }, + { + name: "Failure - Invalid Query", + query: `INVALID QUERY`, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + result, err := provider.runElasticQuery(ctx, tt.query) + + if tt.expectedError { + assert.Error(t, err) + assert.Nil(t, result) + } else { + assert.NoError(t, err) + assert.NotNil(t, result) + + jsonData, _ := json.Marshal(result) + assert.True(t, json.Valid(jsonData)) + } + }) + } +} + +func TestExtractMetric(t *testing.T) { + provider := &KeptnElasticProvider{ + Log: logr.Discard(), + } + + tests := []struct { + name string + result map[string]interface{} + metricPath string + expectedValue string + expectedError bool + }{ + { + name: "Success - Metric Found", + result: map[string]interface{}{ + "metrics": map[string]interface{}{ + "cpu": 75.5, + }, + }, + metricPath: "metrics.cpu", + expectedValue: "75.500000", + expectedError: false, + }, + { + name: "Failure - Metric Not Found", + result: map[string]interface{}{ + "metrics": map[string]interface{}{ + "memory": 1024, + }, + }, + metricPath: "metrics.cpu", + expectedValue: "", + expectedError: false, + }, + { + name: "Success - Nested Metric", + result: map[string]interface{}{ + "root": map[string]interface{}{ + "sub": map[string]interface{}{ + "value": 42.3, + }, + }, + }, + metricPath: "root.sub.value", + expectedValue: "42.300000", + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + value, err := provider.extractMetric(tt.result, tt.metricPath) + + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedValue, value) + } + }) + } +} + +func TestConvertResultTOMap(t *testing.T) { + input := map[string]interface{}{ + "cpu": 50, + "mem": 2048.5, + "disk": map[string]interface{}{ + "used": 500, + "free": 1500.75, + }, + } + expected := map[string]float64{ + "cpu": 50, + "mem": 2048.5, + "disk.used": 500, + "disk.free": 1500.75, + } + + output := convertResultTOMap(input) + assert.Equal(t, expected, output) +} diff --git a/metrics-operator/controllers/common/providers/provider.go b/metrics-operator/controllers/common/providers/provider.go index d57dc32af8..319e00b98c 100644 --- a/metrics-operator/controllers/common/providers/provider.go +++ b/metrics-operator/controllers/common/providers/provider.go @@ -11,6 +11,7 @@ import ( metricsapi "github.com/keptn/lifecycle-toolkit/metrics-operator/api/v1" "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers/datadog" "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers/dynatrace" + "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers/elastic" "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers/prometheus" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -62,6 +63,16 @@ func NewProvider(provider *metricsapi.KeptnMetricsProvider, log logr.Logger, k8s }, K8sClient: k8sClient, }, nil + case ElasticProviderType: + es, err := elastic.GetElasticClient(*provider) + if err != nil { + return nil, err + } + return &elastic.KeptnElasticProvider{ + Log: log, + K8sClient: k8sClient, + Elastic: es, + }, nil default: return nil, fmt.Errorf("provider %s not supported", provider.Spec.Type) } diff --git a/metrics-operator/controllers/common/providers/provider_test.go b/metrics-operator/controllers/common/providers/provider_test.go index 7288bde6ae..137f24f835 100644 --- a/metrics-operator/controllers/common/providers/provider_test.go +++ b/metrics-operator/controllers/common/providers/provider_test.go @@ -8,6 +8,7 @@ import ( "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/fake" "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers/datadog" "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers/dynatrace" + "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers/elastic" "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers/prometheus" "github.com/stretchr/testify/require" ) @@ -45,6 +46,15 @@ func TestFactory(t *testing.T) { provider: &prometheus.KeptnPrometheusProvider{}, err: false, }, + { + metricsProvider: metricsapi.KeptnMetricsProvider{ + Spec: metricsapi.KeptnMetricsProviderSpec{ + Type: ElasticProviderType, + }, + }, + provider: &elastic.KeptnElasticProvider{}, + err: false, + }, { metricsProvider: metricsapi.KeptnMetricsProvider{ Spec: metricsapi.KeptnMetricsProviderSpec{ diff --git a/metrics-operator/go.mod b/metrics-operator/go.mod index c91147dba7..439785dff0 100644 --- a/metrics-operator/go.mod +++ b/metrics-operator/go.mod @@ -5,6 +5,7 @@ go 1.23 require ( github.com/DataDog/datadog-api-client-go/v2 v2.32.0 github.com/benbjohnson/clock v1.3.5 + github.com/elastic/go-elasticsearch/v8 v8.17.1 github.com/go-logr/logr v1.4.2 github.com/gorilla/mux v1.8.1 github.com/kelseyhightower/envconfig v1.4.0 @@ -43,6 +44,7 @@ require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect diff --git a/metrics-operator/go.sum b/metrics-operator/go.sum index 97dc326ad2..f040acbf54 100644 --- a/metrics-operator/go.sum +++ b/metrics-operator/go.sum @@ -29,6 +29,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elastic/elastic-transport-go/v8 v8.6.1 h1:h2jQRqH6eLGiBSN4eZbQnJLtL4bC5b4lfVFRjw2R4e4= +github.com/elastic/elastic-transport-go/v8 v8.6.1/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v8 v8.17.1 h1:bOXChDoCMB4TIwwGqKd031U8OXssmWLT3UrAr9EGs3Q= +github.com/elastic/go-elasticsearch/v8 v8.17.1/go.mod h1:MVJCtL+gJJ7x5jFeUmA20O7rvipX8GcQmo5iBcmaJn4= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=