[9.1] (backport #11394) [otel config translate] allow hosts to be string not just slice (#11431)

* [otel config translate] allow hosts to be string not just slice (#11394)

* [otel config translate] allow hosts to be string not just slice

* update changelog to bugfix

(cherry picked from commit 70ef801)

# Conflicts:
#	internal/pkg/otel/translate/output_elasticsearch.go
#	internal/pkg/otel/translate/output_elasticsearch_test.go

* fix merge conflicts

* remove test cases that shouldn't have been backported

---------

Co-authored-by: Lee E Hinman <[email protected]>
Co-authored-by: Lee E. Hinman <[email protected]>

Author: mergify[bot]

changelog/fragments/1764016303-allow-host-to-be-a-string-for-otel-configuration-translation.yaml

@@ -0,0 +1,46 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: allow host to be a string for otel configuration translation
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# description:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/11394
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/11352
+

internal/pkg/otel/translate/output_elasticsearch.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"reflect"
+	"slices"
 	"strings"
 	"time"
 
@@ -27,7 +28,6 @@ import (
 
 type esToOTelOptions struct {
 	elasticsearch.ElasticsearchConfig `config:",inline"`
-	outputs.HostWorkerCfg             `config:",inline"`
 
 	Index    string `config:"index"`
 	Pipeline string `config:"pipeline"`
@@ -36,13 +36,9 @@ type esToOTelOptions struct {
 
 var defaultOptions = esToOTelOptions{
 	ElasticsearchConfig: elasticsearch.DefaultConfig(),
-
-	Index:    "", // Dynamic routing is disabled if index is set
-	Pipeline: "",
-	Preset:   "custom", // default is custom if not set
-	HostWorkerCfg: outputs.HostWorkerCfg{
-		Workers: 1,
-	},
+	Index:               "", // Dynamic routing is disabled if index is set
+	Pipeline:            "",
+	Preset:              "custom", // default is custom if not set
 }
 
 // ToOTelConfig converts a Beat config into OTel elasticsearch exporter config
@@ -99,13 +95,19 @@ func ToOTelConfig(output *config.C, logger *logp.Logger) (map[string]any, error)
 	}
 
 	// Create url using host name, protocol and path
+	outputHosts, err := outputs.ReadHostList(output)
+	if err != nil {
+		return nil, fmt.Errorf("error reading host list: %w", err)
+	}
 	hosts := []string{}
-	for _, h := range escfg.Hosts {
+	for _, h := range outputHosts {
 		esURL, err := common.MakeURL(escfg.Protocol, escfg.Path, h, 9200)
 		if err != nil {
 			return nil, fmt.Errorf("cannot generate ES URL from host %w", err)
 		}
-		hosts = append(hosts, esURL)
+		if !slices.Contains(hosts, esURL) {
+			hosts = append(hosts, esURL)
+		}
 	}
 
 	otelYAMLCfg := map[string]any{
@@ -116,7 +118,7 @@ func ToOTelConfig(output *config.C, logger *logp.Logger) (map[string]any, error)
 		// where it could spin as many goroutines as it liked.
 		// Given that batcher implementation can change and it has a history of such changes,
 		// let's keep max_conns_per_host setting for now and remove it once exporterhelper is stable.
-		"max_conns_per_host": escfg.NumWorkers(),
+		"max_conns_per_host": getTotalNumWorkers(output), // num_workers * len(hosts) if loadbalance is true
 
 		// Retry
 		"retry": map[string]any{
@@ -137,7 +139,7 @@ func ToOTelConfig(output *config.C, logger *logp.Logger) (map[string]any, error)
 			"queue_size":        getQueueSize(logger, output),
 			"block_on_overflow": true,
 			"wait_for_result":   true,
-			"num_consumers":     escfg.NumWorkers(),
+			"num_consumers":     getTotalNumWorkers(output), // num_workers * len(hosts) if loadbalance is true
 		},
 
 		"mapping": map[string]any{
@@ -171,6 +173,16 @@ func ToOTelConfig(output *config.C, logger *logp.Logger) (map[string]any, error)
 	return otelYAMLCfg, nil
 }
 
+// getTotalNumWorkers returns the number of hosts that beats would
+// have used taking into account hosts, loadbalance and worker
+func getTotalNumWorkers(cfg *config.C) int {
+	hostList, err := outputs.ReadHostList(cfg)
+	if err != nil {
+		return 1
+	}
+	return len(hostList)
+}
+
 // log warning for unsupported config
 func checkUnsupportedConfig(cfg *config.C, logger *logp.Logger) error {
 	if cfg.HasField("indices") {