From 39d4c9a01be4c1bc1268699cf886c241257b4fd7 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Fri, 17 Jan 2025 15:54:43 +0000 Subject: [PATCH 1/2] fix watcher settings & add test for full config --- api/grpc/mpi/v1/command.pb.go | 2 +- api/grpc/mpi/v1/common.pb.go | 2 +- api/grpc/mpi/v1/files.pb.go | 2 +- internal/config/config.go | 6 +- internal/config/config_test.go | 252 +++++++++++++++++----- internal/config/testdata/nginx-agent.conf | 94 ++++++-- internal/config/types.go | 5 - test/types/config.go | 1 - 8 files changed, 285 insertions(+), 79 deletions(-) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 94368f7d3..b3cdd5cf1 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.2 +// protoc-gen-go v1.36.3 // protoc (unknown) // source: mpi/v1/command.proto diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 763aae6c0..d2cb5d35b 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.2 +// protoc-gen-go v1.36.3 // protoc (unknown) // source: mpi/v1/common.proto diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index cd41aa738..2cbfb57ec 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.2 +// protoc-gen-go v1.36.3 // protoc (unknown) // source: mpi/v1/files.proto diff --git a/internal/config/config.go b/internal/config/config.go index 23d6f7a05..dbfa4a958 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -753,13 +753,13 @@ func arePrometheusExportTLSSettingsSet() bool { func resolveWatchers() *Watchers { return &Watchers{ InstanceWatcher: InstanceWatcher{ - MonitoringFrequency: DefInstanceWatcherMonitoringFrequency, + MonitoringFrequency: viperInstance.GetDuration(InstanceWatcherMonitoringFrequencyKey), }, InstanceHealthWatcher: InstanceHealthWatcher{ - MonitoringFrequency: DefInstanceHealthWatcherMonitoringFrequency, + MonitoringFrequency: viperInstance.GetDuration(InstanceHealthWatcherMonitoringFrequencyKey), }, FileWatcher: FileWatcher{ - MonitoringFrequency: DefFileWatcherMonitoringFrequency, + MonitoringFrequency: viperInstance.GetDuration(FileWatcherMonitoringFrequencyKey), }, } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 70501d76f..b1d303ea3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -39,10 +39,6 @@ func TestRegisterConfigFile(t *testing.T) { } func TestResolveConfig(t *testing.T) { - allowedDir := []string{ - "/etc/nginx", "/usr/local/etc/nginx", "/var/run/nginx", - "/usr/share/nginx/modules", "/var/log/nginx", - } viperInstance = viper.NewWithOptions(viper.KeyDelimiter(KeyDelimiter)) err := loadPropertiesFromFile("./testdata/nginx-agent.conf") require.NoError(t, err) @@ -57,50 +53,7 @@ func TestResolveConfig(t *testing.T) { actual, err := ResolveConfig() require.NoError(t, err) - - assert.Equal(t, "debug", actual.Log.Level) - assert.Equal(t, "./", actual.Log.Path) - - assert.Equal(t, 30*time.Second, actual.DataPlaneConfig.Nginx.ReloadMonitoringPeriod) - assert.False(t, actual.DataPlaneConfig.Nginx.TreatWarningsAsErrors) - assert.Equal(t, []string{"/var/log/nginx/error.log", "/var/log/nginx/access.log"}, - actual.DataPlaneConfig.Nginx.ExcludeLogs) - - require.NotNil(t, actual.Collector) - assert.Equal(t, "/etc/nginx-agent/nginx-agent-otelcol.yaml", actual.Collector.ConfigPath) - assert.NotEmpty(t, actual.Collector.Receivers) - assert.Equal(t, Processors{Batch: &Batch{}}, actual.Collector.Processors) - assert.NotEmpty(t, actual.Collector.Exporters) - assert.NotEmpty(t, actual.Collector.Extensions) - - // Client GRPC Settings - assert.Equal(t, 15*time.Second, actual.Client.Grpc.KeepAlive.Timeout) - assert.Equal(t, 10*time.Second, actual.Client.Grpc.KeepAlive.Time) - assert.False(t, actual.Client.Grpc.KeepAlive.PermitWithoutStream) - assert.Equal(t, 1048575, actual.Client.Grpc.MaxMessageSize) - assert.Equal(t, 1048575, actual.Client.Grpc.MaxMessageReceiveSize) - assert.Equal(t, 1048575, actual.Client.Grpc.MaxMessageSendSize) - - // Client HTTP Settings - assert.Equal(t, 15*time.Second, actual.Client.HTTP.Timeout) - - // Client Backoff Settings - assert.Equal(t, 200*time.Millisecond, actual.Client.Backoff.InitialInterval) - assert.Equal(t, 10*time.Second, actual.Client.Backoff.MaxInterval) - assert.Equal(t, 25*time.Second, actual.Client.Backoff.MaxElapsedTime) - assert.InDelta(t, 1.5, actual.Client.Backoff.RandomizationFactor, 0.01) - assert.InDelta(t, 2.5, actual.Client.Backoff.Multiplier, 0.01) - - assert.Equal(t, - allowedDir, - actual.AllowedDirectories, - ) - - assert.Equal(t, allowedDir, actual.AllowedDirectories) - - assert.Equal(t, 5*time.Second, actual.Watchers.InstanceWatcher.MonitoringFrequency) - assert.Equal(t, 5*time.Second, actual.Watchers.InstanceHealthWatcher.MonitoringFrequency) - assert.Equal(t, 5*time.Second, actual.Watchers.FileWatcher.MonitoringFrequency) + assert.Equal(t, createConfig(), actual) } func TestSetVersion(t *testing.T) { @@ -462,3 +415,206 @@ func getAgentConfig() *Config { }, } } + +func createConfig() *Config { + return &Config{ + Log: &Log{ + Level: "debug", + Path: "./test-path", + }, + Client: &Client{ + HTTP: &HTTP{ + Timeout: 15 * time.Second, + }, + Grpc: &GRPC{ + KeepAlive: &KeepAlive{ + Timeout: 15 * time.Second, + Time: 10 * time.Second, + PermitWithoutStream: false, + }, + MaxMessageSize: 1048575, + MaxMessageReceiveSize: 1048575, + MaxMessageSendSize: 1048575, + }, + Backoff: &BackOff{ + InitialInterval: 200 * time.Millisecond, + MaxInterval: 10 * time.Second, + MaxElapsedTime: 25 * time.Second, + RandomizationFactor: 1.5, + Multiplier: 2.5, + }, + }, + AllowedDirectories: []string{ + "/etc/nginx", "/usr/local/etc/nginx", "/var/run/nginx", "/usr/share/nginx/modules", "/var/log/nginx", + }, + DataPlaneConfig: &DataPlaneConfig{ + Nginx: &NginxDataPlaneConfig{ + ExcludeLogs: []string{"/var/log/nginx/error.log", "/var/log/nginx/access.log"}, + ReloadMonitoringPeriod: 30 * time.Second, + TreatWarningsAsErrors: true, + }, + }, + Collector: &Collector{ + ConfigPath: "/etc/nginx-agent/nginx-agent-otelcol.yaml", + Exporters: Exporters{ + OtlpExporters: []OtlpExporter{ + { + Server: &ServerConfig{ + Host: "127.0.0.1", + Port: 5643, + Type: 0, + }, + Authenticator: "test-saas-token", + TLS: &TLSConfig{ + Cert: "/path/to/server-cert.pem", + Key: "/path/to/server-key.pem", + Ca: "/path/to/server-cert.pem", + SkipVerify: false, + ServerName: "test-saas-server", + }, + }, + }, + PrometheusExporter: &PrometheusExporter{ + Server: &ServerConfig{ + Host: "127.0.0.1", + Port: 1235, + Type: 0, + }, + TLS: &TLSConfig{ + Cert: "/path/to/server-cert.pem", + Key: "/path/to/server-key.pem", + Ca: "/path/to/server-cert.pem", + SkipVerify: false, + ServerName: "test-server", + }, + }, + Debug: &DebugExporter{}, + }, + Processors: Processors{ + Batch: &Batch{ + SendBatchMaxSize: 1, + SendBatchSize: 8199, + Timeout: 30 * time.Second, + }, + Attribute: &Attribute{ + Actions: []Action{ + { + Key: "test", + Action: "insert", + Value: "value", + }, + }, + }, + }, + Receivers: Receivers{ + OtlpReceivers: []OtlpReceiver{ + { + Server: &ServerConfig{ + Host: "127.0.0.1", + Port: 4317, + Type: 0, + }, + Auth: &AuthConfig{ + Token: "secret-receiver-token", + }, + OtlpTLSConfig: &OtlpTLSConfig{ + GenerateSelfSignedCert: false, + Cert: "/tmp/cert.pem", + Key: "/tmp/key.pem", + Ca: "/tmp/ca.pem", + SkipVerify: true, + ServerName: "test-local-server", + }, + }, + }, + NginxReceivers: []NginxReceiver{ + { + InstanceID: "cd7b8911-c2c5-4daf-b311-dbead151d938", + AccessLogs: []AccessLog{ + { + LogFormat: "$remote_addr - $remote_user [$time_local] \"$request\"" + + " $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" " + + "\"$http_x_forwarded_for\"", + FilePath: "/var/log/nginx/access-custom.conf", + }, + }, + }, + }, + NginxPlusReceivers: []NginxPlusReceiver{ + { + InstanceID: "cd7b8911-c2c5-4daf-b311-dbead151d939", + }, + }, + HostMetrics: &HostMetrics{ + CollectionInterval: 10 * time.Second, + InitialDelay: 2 * time.Second, + Scrapers: &HostMetricsScrapers{ + CPU: &CPUScraper{}, + Disk: nil, + Filesystem: nil, + Memory: nil, + Network: nil, + }, + }, + }, + Extensions: Extensions{ + Health: &Health{ + Server: &ServerConfig{ + Host: "127.0.0.1", + Port: 1337, + Type: 0, + }, + TLS: &TLSConfig{ + Cert: "/path/to/server-cert.pem", + Key: "/path/to/server-key.pem", + Ca: "/path/to/server-ca.pem", + SkipVerify: false, + ServerName: "server-name", + }, + Path: "/test", + }, + HeadersSetter: &HeadersSetter{ + Headers: []Header{ + { + Action: "action", + Key: "key", + Value: "value", + }, + }, + }, + }, + Log: &Log{ + Level: "INFO", + Path: "/var/log/nginx-agent/opentelemetry-collector-agent.log", + }, + }, + Command: &Command{ + Server: &ServerConfig{ + Host: "127.0.0.1", + Port: 8888, + Type: Grpc, + }, + Auth: &AuthConfig{ + Token: "1234", + }, + TLS: &TLSConfig{ + Cert: "some.cert", + Key: "some.key", + Ca: "some.ca", + SkipVerify: false, + ServerName: "server-name", + }, + }, + Watchers: &Watchers{ + InstanceWatcher: InstanceWatcher{ + MonitoringFrequency: 10 * time.Second, + }, + InstanceHealthWatcher: InstanceHealthWatcher{ + MonitoringFrequency: 10 * time.Second, + }, + FileWatcher: FileWatcher{ + MonitoringFrequency: 10 * time.Second, + }, + }, + } +} diff --git a/internal/config/testdata/nginx-agent.conf b/internal/config/testdata/nginx-agent.conf index e63ece84f..6db2220d5 100644 --- a/internal/config/testdata/nginx-agent.conf +++ b/internal/config/testdata/nginx-agent.conf @@ -1,19 +1,19 @@ log: level: debug - path: ./ + path: ./test-path watchers: instance_watcher: - monitoring_frequency: 5s + monitoring_frequency: 10s instance_health_watcher: - monitoring_frequency: 5s + monitoring_frequency: 10s file_watcher: - monitoring_frequency: 5s + monitoring_frequency: 10s data_plane_config: nginx: reload_monitoring_period: 30s - treat_warnings_as_errors: false + treat_warnings_as_errors: true exclude_logs: - /var/log/nginx/error.log - /var/log/nginx/access.log @@ -34,7 +34,28 @@ client: max_elapsed_time: 25s randomization_factor: 1.5 multiplier: 2.5 - + +allowed_directories: + - /etc/nginx + - /usr/local/etc/nginx + - /var/run/nginx + - /usr/share/nginx/modules + - /var/log/nginx + +command: + server: + host: "127.0.0.1" + port: 8888 + type: 0 + auth: + token: "1234" + tls: + cert: "some.cert" + key: "some.key" + ca: "some.ca" + skip_verify: false + server_name: "server-name" + collector: config_path: "/etc/nginx-agent/nginx-agent-otelcol.yaml" receivers: @@ -48,39 +69,74 @@ collector: tls: generate_self_signed_cert: false server_name: "test-local-server" + skip_verify: true ca: /tmp/ca.pem cert: /tmp/cert.pem key: /tmp/key.pem nginx_receivers: - instance_id: cd7b8911-c2c5-4daf-b311-dbead151d938 - stub_status: "http://localhost:4321/status" access_logs: - file_path: "/var/log/nginx/access-custom.conf" - log_format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\"\"$upstream_cache_status\"" + log_format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\"" + nginx_plus_receivers: + - instance_id: cd7b8911-c2c5-4daf-b311-dbead151d939 + host_metrics: + collection_interval: 10s + initial_delay: 2s + scrapers: + cpu: {} processors: - batch: {} + batch: + send_batch_max_size: 1 + send_batch_size: 8199 + timeout: 30s + attribute: + actions: + - key: "test" + action: "insert" + value: "value" exporters: otlp_exporters: - server: host: "127.0.0.1" port: 5643 - auth: - Token: "secret-saas-token" + authenticator: "test-saas-token" tls: server_name: "test-saas-server" skip_verify: false cert: /path/to/server-cert.pem key: /path/to/server-key.pem ca: /path/to/server-cert.pem + prometheus_exporter: + server: + host: "127.0.0.1" + port: 1235 + type: 0 + tls: + server_name: "test-server" + skip_verify: false + cert: /path/to/server-cert.pem + key: /path/to/server-key.pem + ca: /path/to/server-cert.pem + debug: {} extensions: + headers_setter: + headers: + - action: "action" + key: "key" + value: "value" health: server: host: "127.0.0.1" - port: 1234 - -allowed_directories: - - /etc/nginx - - /usr/local/etc/nginx - - /var/run/nginx - - /usr/share/nginx/modules - - /var/log/nginx + port: 1337 + type: 0 + path: "/test" + tls: + server_name: "server-name" + skip_verify: false + cert: /path/to/server-cert.pem + key: /path/to/server-key.pem + ca: /path/to/server-ca.pem + log: + level: "INFO" + path: "/var/log/nginx-agent/opentelemetry-collector-agent.log" diff --git a/internal/config/types.go b/internal/config/types.go index b02a0602d..e548f9bd3 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -37,7 +37,6 @@ type ( DataPlaneConfig *DataPlaneConfig `yaml:"-" mapstructure:"data_plane_config"` Client *Client `yaml:"-" mapstructure:"client"` Collector *Collector `yaml:"-" mapstructure:"collector"` - File *File `yaml:"-" mapstructure:"file"` Watchers *Watchers `yaml:"-"` Version string `yaml:"-"` Path string `yaml:"-"` @@ -283,10 +282,6 @@ type ( GenerateSelfSignedCert bool `yaml:"-" mapstructure:"generate_self_signed_cert"` } - File struct { - Location string `yaml:"-" mapstructure:"location"` - } - Watchers struct { InstanceWatcher InstanceWatcher `yaml:"-" mapstructure:"instance_watcher"` InstanceHealthWatcher InstanceHealthWatcher `yaml:"-" mapstructure:"instance_health_watcher"` diff --git a/test/types/config.go b/test/types/config.go index 60ec80e37..6d812e836 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -143,7 +143,6 @@ func AgentConfig() *config.Config { ServerName: "test-server", }, }, - File: &config.File{}, DataPlaneConfig: &config.DataPlaneConfig{ Nginx: &config.NginxDataPlaneConfig{ TreatWarningsAsErrors: true, From c157cd69afa10c5f499ac924823e9e1dc1dd0a37 Mon Sep 17 00:00:00 2001 From: Aphral Griffin Date: Mon, 20 Jan 2025 11:51:40 +0000 Subject: [PATCH 2/2] add labels to test --- internal/config/config_test.go | 11 ++++++----- internal/config/testdata/nginx-agent.conf | 10 ++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9589c7634..4bf989646 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -132,7 +132,7 @@ func TestLoadPropertiesFromFile(t *testing.T) { require.NoError(t, err) assert.Equal(t, "debug", viperInstance.GetString(LogLevelKey)) - assert.Equal(t, "./", viperInstance.GetString(LogPathKey)) + assert.Equal(t, "./test-path", viperInstance.GetString(LogPathKey)) assert.Equal(t, 15*time.Second, viperInstance.GetDuration(ClientKeepAliveTimeoutKey)) @@ -772,7 +772,6 @@ func createConfig() *Config { Server: &ServerConfig{ Host: "127.0.0.1", Port: 5643, - Type: 0, }, Authenticator: "test-saas-token", TLS: &TLSConfig{ @@ -788,7 +787,6 @@ func createConfig() *Config { Server: &ServerConfig{ Host: "127.0.0.1", Port: 1235, - Type: 0, }, TLS: &TLSConfig{ Cert: "/path/to/server-cert.pem", @@ -822,7 +820,6 @@ func createConfig() *Config { Server: &ServerConfig{ Host: "127.0.0.1", Port: 4317, - Type: 0, }, Auth: &AuthConfig{ Token: "secret-receiver-token", @@ -872,7 +869,6 @@ func createConfig() *Config { Server: &ServerConfig{ Host: "127.0.0.1", Port: 1337, - Type: 0, }, TLS: &TLSConfig{ Cert: "/path/to/server-cert.pem", @@ -926,5 +922,10 @@ func createConfig() *Config { MonitoringFrequency: 10 * time.Second, }, }, + Labels: map[string]any{ + "label1": "label 1", + "label2": "new-value", + "label3": 123, + }, } } diff --git a/internal/config/testdata/nginx-agent.conf b/internal/config/testdata/nginx-agent.conf index 6db2220d5..f138afac9 100644 --- a/internal/config/testdata/nginx-agent.conf +++ b/internal/config/testdata/nginx-agent.conf @@ -10,6 +10,11 @@ watchers: file_watcher: monitoring_frequency: 10s +labels: + label1: label 1 + label2: new-value + label3: 123 + data_plane_config: nginx: reload_monitoring_period: 30s @@ -46,7 +51,7 @@ command: server: host: "127.0.0.1" port: 8888 - type: 0 + type: grpc auth: token: "1234" tls: @@ -63,7 +68,6 @@ collector: - server: host: "127.0.0.1" port: 4317 - type: 0 auth: Token: "secret-receiver-token" tls: @@ -111,7 +115,6 @@ collector: server: host: "127.0.0.1" port: 1235 - type: 0 tls: server_name: "test-server" skip_verify: false @@ -129,7 +132,6 @@ collector: server: host: "127.0.0.1" port: 1337 - type: 0 path: "/test" tls: server_name: "server-name"