Skip to content

Commit 3b20859

Browse files
aphralGaphralG
committed
add configuration options for reload backoff
1 parent f2958cb commit 3b20859

File tree

8 files changed

+115
-49
lines changed

8 files changed

+115
-49
lines changed

‎internal/config/config.go‎

Lines changed: 59 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -379,28 +379,6 @@ func registerFlags() {
379379
DefManifestDir,
380380
"Specifies the path to the directory containing the manifest files",
381381
)
382-
fs.Duration(
383-
NginxReloadMonitoringPeriodKey,
384-
DefNginxReloadMonitoringPeriod,
385-
"The amount of time to monitor NGINX after a reload of configuration.",
386-
)
387-
fs.Bool(
388-
NginxTreatWarningsAsErrorsKey,
389-
DefTreatErrorsAsWarnings,
390-
"Warning messages in the NGINX errors logs after a NGINX reload will be treated as an error.",
391-
)
392-
393-
fs.String(
394-
NginxApiTlsCa,
395-
DefNginxApiTlsCa,
396-
"The NGINX Plus CA certificate file location needed to call the NGINX Plus API if SSL is enabled.",
397-
)
398-
399-
fs.StringSlice(
400-
NginxExcludeLogsKey, []string{},
401-
"A comma-separated list of one or more NGINX log paths that you want to exclude from metrics "+
402-
"collection or error monitoring. This includes absolute paths or regex patterns",
403-
)
404382

405383
fs.StringSlice(AllowedDirectoriesKey,
406384
DefaultAllowedDirectories(),
@@ -442,6 +420,7 @@ func registerFlags() {
442420
registerAuxiliaryCommandFlags(fs)
443421
registerCollectorFlags(fs)
444422
registerClientFlags(fs)
423+
registerDataPlaneFlags(fs)
445424

446425
fs.SetNormalizeFunc(normalizeFunc)
447426

@@ -456,6 +435,57 @@ func registerFlags() {
456435
})
457436
}
458437

438+
func registerDataPlaneFlags(fs *flag.FlagSet) {
439+
fs.Duration(
440+
NginxReloadMonitoringPeriodKey,
441+
DefNginxReloadMonitoringPeriod,
442+
"The amount of time to monitor NGINX after a reload of configuration.",
443+
)
444+
fs.Bool(
445+
NginxTreatWarningsAsErrorsKey,
446+
DefTreatErrorsAsWarnings,
447+
"Warning messages in the NGINX errors logs after a NGINX reload will be treated as an error.",
448+
)
449+
450+
fs.String(
451+
NginxApiTlsCa,
452+
DefNginxApiTlsCa,
453+
"The NGINX Plus CA certificate file location needed to call the NGINX Plus API if SSL is enabled.",
454+
)
455+
456+
fs.StringSlice(
457+
NginxExcludeLogsKey, []string{},
458+
"A comma-separated list of one or more NGINX log paths that you want to exclude from metrics "+
459+
"collection or error monitoring. This includes absolute paths or regex patterns",
460+
)
461+
462+
// Nginx Reload Backoff Flags
463+
fs.Duration(
464+
NginxReloadBackoffInitialIntervalKey,
465+
DefNginxReloadBackoffInitialInterval,
466+
"The client backoff initial interval, value in seconds")
467+
468+
fs.Duration(
469+
NginxReloadBackoffMaxIntervalKey,
470+
DefNginxReloadBackoffMaxInterval,
471+
"The client backoff max interval, value in seconds")
472+
473+
fs.Duration(
474+
NginxReloadBackoffMaxElapsedTimeKey,
475+
DefNginxReloadBackoffMaxElapsedTime,
476+
"The client backoff max elapsed time, value in seconds")
477+
478+
fs.Float64(
479+
NginxReloadBackoffRandomizationFactorKey,
480+
DefNginxReloadBackoffRandomizationFactor,
481+
"The client backoff randomization factor, value float")
482+
483+
fs.Float64(
484+
NginxReloadBackoffMultiplierKey,
485+
DefNginxReloadBackoffMultiplier,
486+
"The client backoff multiplier, value float")
487+
}
488+
459489
func registerCommonFlags(fs *flag.FlagSet) {
460490
fs.StringToString(
461491
LabelsRootKey,
@@ -906,6 +936,13 @@ func resolveDataPlaneConfig() *DataPlaneConfig {
906936
TreatWarningsAsErrors: viperInstance.GetBool(NginxTreatWarningsAsErrorsKey),
907937
ExcludeLogs: viperInstance.GetStringSlice(NginxExcludeLogsKey),
908938
APITls: TLSConfig{Ca: viperInstance.GetString(NginxApiTlsCa)},
939+
ReloadBackoff: &BackOff{
940+
InitialInterval: viperInstance.GetDuration(NginxReloadBackoffInitialIntervalKey),
941+
MaxInterval: viperInstance.GetDuration(NginxReloadBackoffMaxIntervalKey),
942+
MaxElapsedTime: viperInstance.GetDuration(NginxReloadBackoffMaxElapsedTimeKey),
943+
RandomizationFactor: viperInstance.GetFloat64(NginxReloadBackoffRandomizationFactorKey),
944+
Multiplier: viperInstance.GetFloat64(NginxReloadBackoffMultiplierKey),
945+
},
909946
},
910947
}
911948
}

‎internal/config/config_test.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1101,6 +1101,13 @@ func createConfig() *Config {
11011101
ExcludeLogs: []string{"/var/log/nginx/error.log", "^/var/log/nginx/.*.log$"},
11021102
ReloadMonitoringPeriod: 30 * time.Second,
11031103
TreatWarningsAsErrors: true,
1104+
ReloadBackoff: &BackOff{
1105+
InitialInterval: 100 * time.Millisecond,
1106+
MaxInterval: 20 * time.Second,
1107+
MaxElapsedTime: 15 * time.Second,
1108+
RandomizationFactor: 1.5,
1109+
Multiplier: 1.5,
1110+
},
11041111
},
11051112
},
11061113
Collector: &Collector{

‎internal/config/defaults.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,13 @@ const (
1616
DefTreatErrorsAsWarnings = false
1717
DefNginxApiTlsCa = ""
1818

19+
// Nginx Reload Backoff defaults
20+
DefNginxReloadBackoffInitialInterval = 1 * time.Second
21+
DefNginxReloadBackoffRandomizationFactor = 0.5 // the value is 0 <= and < 1
22+
DefNginxReloadBackoffMultiplier = 5
23+
DefNginxReloadBackoffMaxInterval = 10 * time.Second
24+
DefNginxReloadBackoffMaxElapsedTime = 30 * time.Second
25+
1926
DefCommandServerHostKey = ""
2027
DefCommandServerPortKey = 0
2128
DefCommandServerTypeKey = "grpc"

‎internal/config/flags.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -112,10 +112,16 @@ var (
112112
LogLevelKey = pre(LogLevelRootKey) + "level"
113113
LogPathKey = pre(LogLevelRootKey) + "path"
114114

115-
NginxReloadMonitoringPeriodKey = pre(DataPlaneConfigRootKey, "nginx") + "reload_monitoring_period"
116-
NginxTreatWarningsAsErrorsKey = pre(DataPlaneConfigRootKey, "nginx") + "treat_warnings_as_errors"
117-
NginxExcludeLogsKey = pre(DataPlaneConfigRootKey, "nginx") + "exclude_logs"
118-
NginxApiTlsCa = pre(DataPlaneConfigRootKey, "nginx") + "api_tls_ca"
115+
NginxReloadMonitoringPeriodKey = pre(DataPlaneConfigRootKey, "nginx") + "reload_monitoring_period"
116+
NginxTreatWarningsAsErrorsKey = pre(DataPlaneConfigRootKey, "nginx") + "treat_warnings_as_errors"
117+
NginxReloadBackoffKey = pre(DataPlaneConfigRootKey, "nginx") + "reload_backoff"
118+
NginxReloadBackoffInitialIntervalKey = pre(NginxReloadBackoffKey) + "initial_interval"
119+
NginxReloadBackoffMaxIntervalKey = pre(NginxReloadBackoffKey) + "max_interval"
120+
NginxReloadBackoffMaxElapsedTimeKey = pre(NginxReloadBackoffKey) + "max_elapsed_time"
121+
NginxReloadBackoffRandomizationFactorKey = pre(NginxReloadBackoffKey) + "randomization_factor"
122+
NginxReloadBackoffMultiplierKey = pre(NginxReloadBackoffKey) + "multiplier"
123+
NginxExcludeLogsKey = pre(DataPlaneConfigRootKey, "nginx") + "exclude_logs"
124+
NginxApiTlsCa = pre(DataPlaneConfigRootKey, "nginx") + "api_tls_ca"
119125

120126
FileWatcherMonitoringFrequencyKey = pre(FileWatcherKey) + "monitoring_frequency"
121127
NginxExcludeFilesKey = pre(FileWatcherKey) + "exclude_files"

‎internal/config/testdata/nginx-agent.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ data_plane_config:
3131
exclude_logs:
3232
- /var/log/nginx/error.log
3333
- ^/var/log/nginx/.*.log$
34+
reload_backoff:
35+
initial_interval: 100ms
36+
max_interval: 20s
37+
max_elapsed_time: 15s
38+
randomization_factor: 1.5
39+
multiplier: 1.5
3440
client:
3541
http:
3642
timeout: 15s

‎internal/config/types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ type (
6363
}
6464

6565
NginxDataPlaneConfig struct {
66+
ReloadBackoff *BackOff `yaml:"reload_backoff" mapstructure:"reload_backoff"`
6667
APITls TLSConfig `yaml:"api_tls" mapstructure:"api_tls"`
6768
ExcludeLogs []string `yaml:"exclude_logs" mapstructure:"exclude_logs"`
6869
ReloadMonitoringPeriod time.Duration `yaml:"reload_monitoring_period" mapstructure:"reload_monitoring_period"`

‎internal/resource/nginx_instance_operator.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,11 +119,11 @@ func (i *NginxInstanceOperator) checkWorkers(ctx context.Context, instanceID str
119119
processes []*nginxprocess.Process,
120120
) {
121121
backoffSettings := &config.BackOff{
122-
InitialInterval: i.agentConfig.Client.Backoff.InitialInterval,
123-
MaxInterval: i.agentConfig.Client.Backoff.MaxInterval,
124-
MaxElapsedTime: i.agentConfig.Client.Backoff.MaxElapsedTime,
125-
RandomizationFactor: i.agentConfig.Client.Backoff.RandomizationFactor,
126-
Multiplier: i.agentConfig.Client.Backoff.Multiplier,
122+
InitialInterval: i.agentConfig.DataPlaneConfig.Nginx.ReloadBackoff.InitialInterval,
123+
MaxInterval: i.agentConfig.DataPlaneConfig.Nginx.ReloadBackoff.MaxInterval,
124+
MaxElapsedTime: i.agentConfig.DataPlaneConfig.Nginx.ReloadBackoff.MaxElapsedTime,
125+
RandomizationFactor: i.agentConfig.DataPlaneConfig.Nginx.ReloadBackoff.RandomizationFactor,
126+
Multiplier: i.agentConfig.DataPlaneConfig.Nginx.ReloadBackoff.Multiplier,
127127
}
128128

129129
slog.DebugContext(ctx, "Waiting for NGINX to finish reloading")
@@ -139,6 +139,7 @@ func (i *NginxInstanceOperator) checkWorkers(ctx context.Context, instanceID str
139139
slog.DebugContext(ctx, "Found parent process ID", "process_id", newPid)
140140

141141
err := backoff.WaitUntil(ctx, backoffSettings, func() error {
142+
slog.Info(" ============ Checking NGINX worker processes have reloaded")
142143
currentWorkers := i.nginxProcessOperator.NginxWorkerProcesses(ctx, newPid)
143144
if len(currentWorkers) == 0 {
144145
return errors.New("waiting for NGINX worker processes")

‎internal/resource/nginx_instance_operator_test.go‎

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -166,12 +166,12 @@ func TestInstanceOperator_Reload(t *testing.T) {
166166
instance := protos.NginxOssInstance([]string{})
167167

168168
agentConfig := types.AgentConfig()
169-
agentConfig.Client.Backoff = &config.BackOff{
170-
InitialInterval: 1 * time.Millisecond,
171-
MaxInterval: 1 * time.Millisecond,
172-
MaxElapsedTime: 1 * time.Second,
173-
RandomizationFactor: 1,
174-
Multiplier: 1,
169+
agentConfig.DataPlaneConfig.Nginx.ReloadBackoff = &config.BackOff{
170+
InitialInterval: config.DefNginxReloadBackoffInitialInterval,
171+
MaxInterval: config.DefNginxReloadBackoffMaxInterval,
172+
MaxElapsedTime: config.DefNginxReloadBackoffMaxElapsedTime,
173+
RandomizationFactor: config.DefNginxReloadBackoffRandomizationFactor,
174+
Multiplier: config.DefNginxReloadBackoffMultiplier,
175175
}
176176
operator := NewInstanceOperator(agentConfig)
177177
operator.executer = mockExec
@@ -226,12 +226,12 @@ func TestInstanceOperator_ReloadAndMonitor(t *testing.T) {
226226

227227
agentConfig := types.AgentConfig()
228228
agentConfig.DataPlaneConfig.Nginx.ReloadMonitoringPeriod = 10 * time.Second
229-
agentConfig.Client.Backoff = &config.BackOff{
230-
InitialInterval: 1 * time.Millisecond,
231-
MaxInterval: 1 * time.Millisecond,
232-
MaxElapsedTime: 1 * time.Second,
233-
RandomizationFactor: 1,
234-
Multiplier: 1,
229+
agentConfig.DataPlaneConfig.Nginx.ReloadBackoff = &config.BackOff{
230+
InitialInterval: config.DefNginxReloadBackoffInitialInterval,
231+
MaxInterval: config.DefNginxReloadBackoffMaxInterval,
232+
MaxElapsedTime: config.DefNginxReloadBackoffMaxElapsedTime,
233+
RandomizationFactor: config.DefNginxReloadBackoffRandomizationFactor,
234+
Multiplier: config.DefNginxReloadBackoffMultiplier,
235235
}
236236
operator := NewInstanceOperator(agentConfig)
237237
operator.executer = mockExec
@@ -366,12 +366,12 @@ func TestInstanceOperator_checkWorkers(t *testing.T) {
366366

367367
agentConfig := types.AgentConfig()
368368
agentConfig.DataPlaneConfig.Nginx.ReloadMonitoringPeriod = 10 * time.Second
369-
agentConfig.Client.Backoff = &config.BackOff{
370-
InitialInterval: config.DefBackoffInitialInterval,
371-
MaxInterval: config.DefBackoffMaxInterval,
372-
MaxElapsedTime: 10 * time.Second,
373-
RandomizationFactor: config.DefBackoffRandomizationFactor,
374-
Multiplier: config.DefBackoffMultiplier,
369+
agentConfig.DataPlaneConfig.Nginx.ReloadBackoff = &config.BackOff{
370+
InitialInterval: config.DefNginxReloadBackoffInitialInterval,
371+
MaxInterval: config.DefNginxReloadBackoffMaxInterval,
372+
MaxElapsedTime: config.DefNginxReloadBackoffMaxElapsedTime,
373+
RandomizationFactor: config.DefNginxReloadBackoffRandomizationFactor,
374+
Multiplier: config.DefNginxReloadBackoffMultiplier,
375375
}
376376
operator := NewInstanceOperator(agentConfig)
377377
operator.executer = mockExec
@@ -381,6 +381,7 @@ func TestInstanceOperator_checkWorkers(t *testing.T) {
381381

382382
helpers.ValidateLog(t, test.expectedLog, logBuf)
383383

384+
t.Logf("Logs: %s", logBuf.String())
384385
logBuf.Reset()
385386
})
386387
}

0 commit comments

Comments
 (0)