Fix backoff to drop Metrics Reports from buffer after max_elapsed_time has been reached (nginx#752)

* separate backoff settings for metrics

Author: oliveromahony

site/content/configuration/configuration-overview.md

@@ -51,6 +51,12 @@ server:
   # host of the control plane
   host: <FQDN>
   grpcPort: 443
+  backoff: # note: default values are prepopulated 
+    initial_interval: 100ms # Add the appropriate duration value here, e.g., "100ms" for 100 milliseconds, "5s" for 5 seconds, "1m" for 1 minute, "1h" for 1 hour
+    randomization_factor: 0.10 # Add the appropriate float value here, e.g., 0.10
+    multiplier: 1.5 # Add the appropriate float value here, e.g., 1.5
+    max_interval: 1m # Add the appropriate duration value here, e.g., "100ms" for 100 milliseconds, "5s" for 5 seconds, "1m" for 1 minute, "1h" for 1 hour
+    max_elapsed_time: 0 # Add the appropriate duration value here, e.g., "0" for indefinite "100ms" for 100 milliseconds, "5s" for 5 seconds, "1m" for 1 minute, "1h" for 1 hour
 # tls options
 tls:
   # enable tls in the nginx-agent setup for grpcs
@@ -83,6 +89,12 @@ metrics:
   report_interval: 1m
   collection_interval: 15s
   mode: aggregated
+    backoff: # note: default values are prepopulated 
+      initial_interval: 100ms # Add the appropriate duration value here, e.g., "100ms" for 100 milliseconds, "5s" for 5 seconds, "1m" for 1 minute, "1h" for 1 hour
+      randomization_factor: 0.10 # Add the appropriate float value here, e.g., 0.10
+      multiplier: 1.5 # Add the appropriate float value here, e.g., 1.5
+      max_interval: 1m # Add the appropriate duration value here, e.g., "100ms" for 100 milliseconds, "5s" for 5 seconds, "1m" for 1 minute, "1h" for 1 hour
+      max_elapsed_time: 0 # Add the appropriate duration value here, e.g., "0" for indefinite "100ms" for 100 milliseconds, "5s" for 5 seconds, "1m" for 1 minute, "1h" for 1 hour
 
 # OSS NGINX default config path
 # path to aux file dirs can also be added

src/core/config/config.go

@@ -95,6 +95,12 @@ func SetDefaults() {
 	Viper.SetDefault(BackoffMultiplier, Defaults.Server.Backoff.Multiplier)
 	Viper.SetDefault(BackoffRandomizationFactor, Defaults.Server.Backoff.RandomizationFactor)
 
+	Viper.SetDefault(BackoffMetricsInitialInterval, Defaults.AgentMetrics.Backoff.InitialInterval)
+	Viper.SetDefault(BackoffMetricsMaxElapsedTime, Defaults.AgentMetrics.Backoff.MaxElapsedTime)
+	Viper.SetDefault(BackoffMetricsMaxInterval, Defaults.AgentMetrics.Backoff.MaxInterval)
+	Viper.SetDefault(BackoffMetricsMultiplier, Defaults.AgentMetrics.Backoff.Multiplier)
+	Viper.SetDefault(BackoffMetricsRandomizationFactor, Defaults.AgentMetrics.Backoff.RandomizationFactor)
+
 	// DATAPLANE DEFAULTS
 	Viper.SetDefault(DataplaneStatusPoll, Defaults.Dataplane.Status.PollInterval)
 
@@ -133,6 +139,7 @@ func deprecateFlags() {
 	setFlagDeprecated("nap-monitoring-processor-buffer-size", "DEPRECATED. No replacement command.")
 	setFlagDeprecated("nap-monitoring-syslog-ip", "DEPRECATED. No replacement command.")
 	setFlagDeprecated("nap-monitoring-syslog-port", "DEPRECATED. No replacement command.")
+	setFlagDeprecated("metrics-bulksize", "DEPRECATED. Use metrics backoff maxElapsedTime instead to set a time period where no successful connection, after time elapsed start dropping oldest metrics from the buffer.")
 }
 
 func RegisterFlags() {
@@ -315,6 +322,7 @@ func getMetrics() AgentMetrics {
 		ReportInterval:     Viper.GetDuration(MetricsReportInterval),
 		CollectionInterval: Viper.GetDuration(MetricsCollectionInterval),
 		Mode:               Viper.GetString(MetricsMode),
+		Backoff:            getMetricsBackOff(),
 	}
 }
 
@@ -366,6 +374,16 @@ func getBackOff() Backoff {
 	}
 }
 
+func getMetricsBackOff() Backoff {
+	return Backoff{
+		InitialInterval:     Viper.GetDuration(BackoffMetricsInitialInterval),
+		RandomizationFactor: Viper.GetFloat64(BackoffMetricsRandomizationFactor),
+		Multiplier:          Viper.GetFloat64(BackoffMetricsMultiplier),
+		MaxInterval:         Viper.GetDuration(BackoffMetricsMaxInterval),
+		MaxElapsedTime:      Viper.GetDuration(BackoffMetricsMaxElapsedTime),
+	}
+}
+
 func getAgentAPI() AgentAPI {
 	return AgentAPI{
 		Host: Viper.GetString(AgentAPIHost),

src/core/config/config_test.go

@@ -15,6 +15,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 
 	agent_config "github.com/nginx/agent/sdk/v2/agent/config"
 	sysutils "github.com/nginx/agent/v2/test/utils/system"
@@ -256,6 +257,7 @@ func TestGetConfig(t *testing.T) {
 		assert.Equal(t, updatedServerGrpcPort, config.Server.GrpcPort)
 		assert.Equal(t, updatedAgentAPIPort, config.AgentAPI.Port)
 		assert.Equal(t, updatedConfTags, config.Tags)
+		assert.Equal(t, time.Hour, config.Server.Backoff.MaxElapsedTime)
 
 		// Check for updated values
 		assert.Equal(t, updatedConfigDirs, config.ConfigDirs)
@@ -270,6 +272,7 @@ func TestGetConfig(t *testing.T) {
 
 		// Everything else should still be default
 		assert.Equal(t, Defaults.AgentMetrics.Mode, config.AgentMetrics.Mode)
+		assert.Equal(t, 10*time.Minute, config.AgentMetrics.Backoff.MaxInterval)
 		assert.Equal(t, Defaults.Features, config.Features)
 		assert.Equal(t, []string{}, config.Extensions)
 	})

src/core/config/defaults.go

@@ -85,6 +85,13 @@ var (
 			ReportInterval:     1 * time.Minute,
 			CollectionInterval: 15 * time.Second,
 			Mode:               "aggregated",
+			Backoff: Backoff{
+				InitialInterval:     client.DefaultBackoffSettings.InitialInterval,
+				RandomizationFactor: client.DefaultBackoffSettings.Jitter,
+				Multiplier:          client.DefaultBackoffSettings.Multiplier,
+				MaxInterval:         client.DefaultBackoffSettings.MaxInterval,
+				MaxElapsedTime:      client.DefaultBackoffSettings.MaxElapsedTime,
+			},
 		},
 		AgentAPI: AgentAPI{
 			Host: "127.0.0.1",
@@ -184,6 +191,13 @@ const (
 	MetricsReportInterval     = MetricsKey + agent_config.KeyDelimiter + "report_interval"
 	MetricsCollectionInterval = MetricsKey + agent_config.KeyDelimiter + "collection_interval"
 	MetricsMode               = MetricsKey + agent_config.KeyDelimiter + "mode"
+	MetricsBackoff            = MetricsKey + agent_config.KeyDelimiter + "backoff"
+	// metrics backoff settings
+	BackoffMetricsInitialInterval     = MetricsKey + agent_config.KeyDelimiter + BackoffKey + agent_config.KeyDelimiter + "initial_interval"
+	BackoffMetricsRandomizationFactor = MetricsKey + agent_config.KeyDelimiter + BackoffKey + agent_config.KeyDelimiter + "randomization_factor"
+	BackoffMetricsMultiplier          = MetricsKey + agent_config.KeyDelimiter + BackoffKey + agent_config.KeyDelimiter + "multiplier"
+	BackoffMetricsMaxInterval         = MetricsKey + agent_config.KeyDelimiter + BackoffKey + agent_config.KeyDelimiter + "max_interval"
+	BackoffMetricsMaxElapsedTime      = MetricsKey + agent_config.KeyDelimiter + BackoffKey + agent_config.KeyDelimiter + "max_elapsed_time"
 
 	// DEPRECATED KEYS
 	AdvancedMetricsKey                  = "advanced_metrics"

src/core/config/types.go

@@ -70,6 +70,16 @@ func (c *Config) GetServerBackoffSettings() backoff.BackoffSettings {
 	}
 }
 
+func (c *Config) GetMetricsBackoffSettings() backoff.BackoffSettings {
+	return backoff.BackoffSettings{
+		InitialInterval: c.AgentMetrics.Backoff.InitialInterval,
+		MaxInterval:     c.AgentMetrics.Backoff.MaxInterval,
+		MaxElapsedTime:  c.AgentMetrics.Backoff.MaxElapsedTime,
+		Multiplier:      c.AgentMetrics.Backoff.Multiplier,
+		Jitter:          c.AgentMetrics.Backoff.RandomizationFactor,
+	}
+}
+
 type Server struct {
 	Host     string `mapstructure:"host" yaml:"-"`
 	GrpcPort int    `mapstructure:"grpcPort" yaml:"-"`
@@ -137,4 +147,5 @@ type AgentMetrics struct {
 	ReportInterval     time.Duration `mapstructure:"report_interval" yaml:"-"`
 	CollectionInterval time.Duration `mapstructure:"collection_interval" yaml:"-"`
 	Mode               string        `mapstructure:"mode" yaml:"-"`
+	Backoff            Backoff       `mapstructure:"backoff" yaml:"-"`
 }

src/core/grpc.go

@@ -57,7 +57,7 @@ func CreateGrpcClients(ctx context.Context, loadedConfig *config.Config) (client
 	commander.WithDialOptions(append(grpcDialOptions, secureCmdDialOpts)...)
 
 	reporter := client.NewMetricReporterClient()
-	reporter.WithBackoffSettings(loadedConfig.GetServerBackoffSettings())
+	reporter.WithBackoffSettings(loadedConfig.GetMetricsBackoffSettings())
 	reporter.WithServer(loadedConfig.Server.Target)
 	reporter.WithDialOptions(append(grpcDialOptions, secureMetricsDialOpts)...)
 

src/plugins/testdata/configs/updated.conf

@@ -1,6 +1,12 @@
 server: 
   host: 192.168.0.1
   grpcPort: 11000
+  backoff: 
+    initial_interval: 3s
+    randomization_factor: 0.5
+    multiplier: 1.5
+    max_interval: 10m
+    max_elapsed_time: 1h 
 api:
   port: 9010
 config_dirs: /usr/local/etc/nginx
@@ -9,3 +15,10 @@ log:
   path: ./test-path
 nginx:
   metrics_poll_interval: 3s
+metrics:
+  backoff: 
+    initial_interval: 3s
+    randomization_factor: 0.5
+    multiplier: 1.5
+    max_interval: 10m
+    max_elapsed_time: 1h