[8.19](backport #46846) fix: add retry for listener creation in libbeat (#46862)

* fix: add retry for listener creation in libbeat (#46846)

* fix: add retry for listener creation in libbeat

* rename

* context

* remove deprecated

* cleanup

* cleanup

* retry at API level instead of listener

(cherry picked from commit 32b5e5a)

# Conflicts:
#	x-pack/libbeat/cmd/instance/receiver.go

* merge conflicts

---------

Co-authored-by: Vihas Makwana <[email protected]>
Co-authored-by: Vihas <[email protected]>

Author: mergify[bot]

x-pack/filebeat/fbreceiver/factory.go

@@ -30,7 +30,7 @@ func createDefaultConfig() component.Config {
 	return &Config{}
 }
 
-func createReceiver(_ context.Context, set receiver.Settings, baseCfg component.Config, consumer consumer.Logs) (receiver.Logs, error) {
+func createReceiver(ctx context.Context, set receiver.Settings, baseCfg component.Config, consumer consumer.Logs) (receiver.Logs, error) {
 	cfg, ok := baseCfg.(*Config)
 	if !ok {
 		return nil, fmt.Errorf("could not convert otel config to filebeat config")
@@ -51,7 +51,7 @@ func createReceiver(_ context.Context, set receiver.Settings, baseCfg component.
 	}
 
 	beatCreator := beater.New(inputs.Init)
-	br, err := xpInstance.NewBeatReceiver(b, beatCreator)
+	br, err := xpInstance.NewBeatReceiver(ctx, b, beatCreator)
 	if err != nil {
 		return nil, fmt.Errorf("error creating %s:%w", Name, err)
 	}

x-pack/libbeat/cmd/instance/receiver.go

@@ -5,14 +5,17 @@
 package instance
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/elastic/beats/v7/libbeat/api"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/cmd/instance"
+	"github.com/elastic/beats/v7/libbeat/common/backoff"
 	"github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/otelmanager"
 	"github.com/elastic/beats/v7/x-pack/libbeat/common/otelbeat/status"
 	_ "github.com/elastic/beats/v7/x-pack/libbeat/include"
@@ -31,7 +34,7 @@ type BeatReceiver struct {
 }
 
 // NewBeatReceiver creates a BeatReceiver.  This will also create the beater and start the monitoring server if configured
-func NewBeatReceiver(b *instance.Beat, creator beat.Creator) (BeatReceiver, error) {
+func NewBeatReceiver(ctx context.Context, b *instance.Beat, creator beat.Creator) (BeatReceiver, error) {
 	beatConfig, err := b.BeatConfig()
 	if err != nil {
 		return BeatReceiver{}, fmt.Errorf("error getting beat config: %w", err)
@@ -59,18 +62,25 @@ func NewBeatReceiver(b *instance.Beat, creator beat.Creator) (BeatReceiver, erro
 	}
 
 	if b.Config.HTTP.Enabled() {
-		var err error
-		b.API, err = api.NewWithDefaultRoutes(
-			logp.L().Named("metrics.http"),
-			b.Config.HTTP,
-			b.Monitoring.InfoRegistry(),
-			b.Monitoring.StateRegistry(),
-			b.Monitoring.StatsRegistry(),
-			b.Monitoring.InputsRegistry())
+		retryer := backoff.NewRetryer(50, 100*time.Millisecond, 1*time.Second)
+		err := retryer.Retry(ctx, func() error {
+			var err error
+			b.API, err = api.NewWithDefaultRoutes(
+				logp.L().Named("metrics.http"),
+				b.Config.HTTP,
+				b.Monitoring.InfoRegistry(),
+				b.Monitoring.StateRegistry(),
+				b.Monitoring.StatsRegistry(),
+				b.Monitoring.InputsRegistry())
+			if err != nil {
+				return fmt.Errorf("could not start the HTTP server for the API: %w", err)
+			}
+			b.API.Start()
+			return nil
+		})
 		if err != nil {
-			return BeatReceiver{}, fmt.Errorf("could not start the HTTP server for the API: %w", err)
+			return BeatReceiver{}, fmt.Errorf("error creating api listener after 100 retries: %w", err)
 		}
-		b.API.Start()
 	}
 
 	beater, err := creator(&b.Beat, beatConfig)

x-pack/metricbeat/mbreceiver/factory.go

@@ -37,7 +37,7 @@ func createDefaultConfig() component.Config {
 	return &Config{}
 }
 
-func createReceiver(_ context.Context, set receiver.Settings, baseCfg component.Config, consumer consumer.Logs) (receiver.Logs, error) {
+func createReceiver(ctx context.Context, set receiver.Settings, baseCfg component.Config, consumer consumer.Logs) (receiver.Logs, error) {
 	cfg, ok := baseCfg.(*Config)
 	if !ok {
 		return nil, fmt.Errorf("could not convert otel config to metricbeat config")
@@ -57,7 +57,7 @@ func createReceiver(_ context.Context, set receiver.Settings, baseCfg component.
 	}
 
 	beatCreator := beater.DefaultCreator()
-	br, err := xpInstance.NewBeatReceiver(b, beatCreator)
+	br, err := xpInstance.NewBeatReceiver(ctx, b, beatCreator)
 	if err != nil {
 		return nil, fmt.Errorf("error creating %s: %w", Name, err)
 	}