NGINX Configs to Reference Remote External Files (#1389)

Added support to reference external resources in config apply

Author: Akshay2191

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/docker/docker v28.5.2+incompatible
 	github.com/fsnotify/fsnotify v1.9.0
+	github.com/gabriel-vasile/mimetype v1.4.8
 	github.com/go-resty/resty/v2 v2.16.5
 	github.com/goccy/go-yaml v1.18.0
 	github.com/google/go-cmp v0.7.0
@@ -151,7 +152,6 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/foxboron/go-tpm-keyfiles v0.0.0-20250903184740-5d135037bd4d // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/gin-contrib/sse v1.1.0 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

internal/config/config.go

@@ -50,6 +50,10 @@ const (
 	regexLabelPattern = "^[a-zA-Z0-9]([a-zA-Z0-9-_.]{0,254}[a-zA-Z0-9])?$"
 )
 
+var domainRegex = regexp.MustCompile(
+	`^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$`,
+)
+
 var viperInstance = viper.NewWithOptions(viper.KeyDelimiter(KeyDelimiter))
 
 func RegisterRunner(r func(cmd *cobra.Command, args []string)) {
@@ -158,6 +162,7 @@ func ResolveConfig() (*Config, error) {
 		Labels:             resolveLabels(),
 		LibDir:             viperInstance.GetString(LibDirPathKey),
 		SyslogServer:       resolveSyslogServer(),
+		ExternalDataSource: resolveExternalDataSource(),
 	}
 
 	defaultCollector(collector, config)
@@ -475,6 +480,7 @@ func registerFlags() {
 	registerCollectorFlags(fs)
 	registerClientFlags(fs)
 	registerDataPlaneFlags(fs)
+	registerExternalDataSourceFlags(fs)
 
 	fs.SetNormalizeFunc(normalizeFunc)
 
@@ -489,6 +495,29 @@ func registerFlags() {
 	})
 }
 
+func registerExternalDataSourceFlags(fs *flag.FlagSet) {
+	fs.String(
+		ExternalDataSourceProxyUrlKey,
+		DefExternalDataSourceProxyUrl,
+		"Url to the proxy service for fetching external files.",
+	)
+	fs.StringSlice(
+		ExternalDataSourceAllowDomainsKey,
+		[]string{},
+		"List of allowed domains for external data sources.",
+	)
+	fs.StringSlice(
+		ExternalDataSourceAllowedFileTypesKey,
+		[]string{},
+		"List of allowed file types for external data sources.",
+	)
+	fs.Int64(
+		ExternalDataSourceMaxBytesKey,
+		DefExternalDataSourceMaxBytes,
+		"Maximum size in bytes for external data sources.",
+	)
+}
+
 func registerDataPlaneFlags(fs *flag.FlagSet) {
 	fs.Duration(
 		NginxReloadMonitoringPeriodKey,
@@ -646,6 +675,11 @@ func registerClientFlags(fs *flag.FlagSet) {
 		DefMaxParallelFileOperations,
 		"Maximum number of file downloads or uploads performed in parallel",
 	)
+	fs.Duration(
+		ClientFileDownloadTimeoutKey,
+		DefClientFileDownloadTimeout,
+		"Timeout value in seconds, for downloading a file during a config apply.",
+	)
 }
 
 func registerCommandFlags(fs *flag.FlagSet) {
@@ -1134,6 +1168,7 @@ func resolveClient() *Client {
 			RandomizationFactor: viperInstance.GetFloat64(ClientBackoffRandomizationFactorKey),
 			Multiplier:          viperInstance.GetFloat64(ClientBackoffMultiplierKey),
 		},
+		FileDownloadTimeout: viperInstance.GetDuration(ClientFileDownloadTimeoutKey),
 	}
 }
 
@@ -1574,3 +1609,37 @@ func areCommandServerProxyTLSSettingsSet() bool {
 		viperInstance.IsSet(CommandServerProxyTLSSkipVerifyKey) ||
 		viperInstance.IsSet(CommandServerProxyTLSServerNameKey)
 }
+
+func resolveExternalDataSource() *ExternalDataSource {
+	proxyURLStruct := ProxyURL{
+		URL: viperInstance.GetString(ExternalDataSourceProxyUrlKey),
+	}
+	externalDataSource := &ExternalDataSource{
+		ProxyURL:         proxyURLStruct,
+		AllowedDomains:   viperInstance.GetStringSlice(ExternalDataSourceAllowDomainsKey),
+		AllowedFileTypes: viperInstance.GetStringSlice(ExternalDataSourceAllowedFileTypesKey),
+		MaxBytes:         viperInstance.GetInt64(ExternalDataSourceMaxBytesKey),
+	}
+
+	if err := validateAllowedDomains(externalDataSource.AllowedDomains); err != nil {
+		slog.Error("External data source not configured due to invalid configuration", "error", err)
+		return nil
+	}
+
+	return externalDataSource
+}
+
+func validateAllowedDomains(domains []string) error {
+	if len(domains) == 0 {
+		return nil
+	}
+
+	for _, domain := range domains {
+		// Validating syntax using the RFC-compliant regex
+		if !domainRegex.MatchString(domain) || domain == "" {
+			return errors.New("invalid domain found in allowed_domains")
+		}
+	}
+
+	return nil
+}

internal/config/config_test.go

@@ -1161,6 +1161,7 @@ func agentConfig() *Config {
 	}
 }
 
+//nolint:maintidx // createConfig creates a sample Config object for testing purposes.
 func createConfig() *Config {
 	return &Config{
 		Log: &Log{
@@ -1428,6 +1429,13 @@ func createConfig() *Config {
 			config.FeatureCertificates, config.FeatureFileWatcher, config.FeatureMetrics,
 			config.FeatureAPIAction, config.FeatureLogsNap,
 		},
+		ExternalDataSource: &ExternalDataSource{
+			ProxyURL: ProxyURL{
+				URL: "http://proxy.example.com",
+			},
+			AllowedDomains: []string{"example.com", "api.example.com"},
+			MaxBytes:       1048576,
+		},
 	}
 }
 
@@ -1622,3 +1630,64 @@ func TestValidateLabel(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateAllowedDomains(t *testing.T) {
+	tests := []struct {
+		expectedErr error
+		name        string
+		domains     []string
+	}{
+		{
+			name:        "Test 1: Success: Empty slice",
+			domains:     []string{},
+			expectedErr: nil,
+		},
+		{
+			name:        "Test 2: Success: Nil slice",
+			domains:     nil,
+			expectedErr: nil,
+		},
+		{
+			name:        "Test 3: Success: Valid domains",
+			domains:     []string{"example.com", "api.nginx.com", "sub.domain.io"},
+			expectedErr: nil,
+		},
+		{
+			name:        "Test 4: Failure: Domain contains space",
+			domains:     []string{"valid.com", "bad domain.com"},
+			expectedErr: errors.New("invalid domain found in allowed_domains"),
+		},
+		{
+			name:        "Test 5: Failure: Empty string domain",
+			domains:     []string{"valid.com", ""},
+			expectedErr: errors.New("invalid domain found in allowed_domains"),
+		},
+		{
+			name:        "Test 6: Failure: Domain contains forward slash /",
+			domains:     []string{"domain.com/path"},
+			expectedErr: errors.New("invalid domain found in allowed_domains"),
+		},
+		{
+			name:        "Test 7: Failure: Domain contains backward slash \\",
+			domains:     []string{"domain.com\\path"},
+			expectedErr: errors.New("invalid domain found in allowed_domains"),
+		},
+		{
+			name:        "Test 8: Failure: Mixed valid and invalid (first is invalid)",
+			domains:     []string{" only.com", "good.com"},
+			expectedErr: errors.New("invalid domain found in allowed_domains"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateAllowedDomains(tt.domains)
+			if tt.expectedErr == nil {
+				assert.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				assert.EqualError(t, err, tt.expectedErr.Error())
+			}
+		})
+	}
+}

internal/config/defaults.go

@@ -84,6 +84,8 @@ const (
 	DefBackoffMaxInterval         = 20 * time.Second
 	DefBackoffMaxElapsedTime      = 1 * time.Minute
 
+	DefClientFileDownloadTimeout = 60 * time.Second
+
 	// Watcher defaults
 	DefInstanceWatcherMonitoringFrequency       = 5 * time.Second
 	DefInstanceHealthWatcherMonitoringFrequency = 5 * time.Second
@@ -116,6 +118,9 @@ const (
 
 	// File defaults
 	DefLibDir = "/var/lib/nginx-agent"
+
+	DefExternalDataSourceProxyUrl = ""
+	DefExternalDataSourceMaxBytes = 100 * 1024 * 1024 // default 100MB
 )
 
 func DefaultFeatures() []string {

internal/config/flags.go

@@ -25,6 +25,7 @@ const (
 	InstanceHealthWatcherMonitoringFrequencyKey = "watchers_instance_health_watcher_monitoring_frequency"
 	FileWatcherKey                              = "watchers_file_watcher"
 	LibDirPathKey                               = "lib_dir"
+	ExternalDataSourceRootKey                   = "external_data_source"
 )
 
 var (
@@ -49,6 +50,7 @@ var (
 	ClientBackoffMaxElapsedTimeKey      = pre(ClientRootKey) + "backoff_max_elapsed_time"
 	ClientBackoffRandomizationFactorKey = pre(ClientRootKey) + "backoff_randomization_factor"
 	ClientBackoffMultiplierKey          = pre(ClientRootKey) + "backoff_multiplier"
+	ClientFileDownloadTimeoutKey        = pre(ClientRootKey) + "file_download_timeout"
 
 	CollectorConfigPathKey                      = pre(CollectorRootKey) + "config_path"
 	CollectorAdditionalConfigPathsKey           = pre(CollectorRootKey) + "additional_config_paths"
@@ -143,6 +145,12 @@ var (
 
 	FileWatcherMonitoringFrequencyKey = pre(FileWatcherKey) + "monitoring_frequency"
 	NginxExcludeFilesKey              = pre(FileWatcherKey) + "exclude_files"
+
+	ExternalDataSourceProxyKey            = pre(ExternalDataSourceRootKey) + "proxy"
+	ExternalDataSourceProxyUrlKey         = pre(ExternalDataSourceProxyKey) + "url"
+	ExternalDataSourceMaxBytesKey         = pre(ExternalDataSourceRootKey) + "max_bytes"
+	ExternalDataSourceAllowDomainsKey     = pre(ExternalDataSourceRootKey) + "allowed_domains"
+	ExternalDataSourceAllowedFileTypesKey = pre(ExternalDataSourceRootKey) + "allowed_file_types"
 )
 
 func pre(prefixes ...string) string {

internal/config/testdata/nginx-agent.conf

@@ -184,3 +184,11 @@ collector:
   log:      
     level: "INFO"
     path: "/var/log/nginx-agent/opentelemetry-collector-agent.log"
+
+external_data_source:
+  proxy:
+    url: "http://proxy.example.com"
+  allowed_domains:
+    - example.com
+    - api.example.com
+  max_bytes: 1048576

internal/config/types.go

@@ -36,21 +36,22 @@ func parseServerType(str string) (ServerType, bool) {
 
 type (
 	Config struct {
-		Command            *Command         `yaml:"command"             mapstructure:"command"`
-		AuxiliaryCommand   *Command         `yaml:"auxiliary_command"   mapstructure:"auxiliary_command"`
-		Log                *Log             `yaml:"log"                 mapstructure:"log"`
-		DataPlaneConfig    *DataPlaneConfig `yaml:"data_plane_config"   mapstructure:"data_plane_config"`
-		Client             *Client          `yaml:"client"              mapstructure:"client"`
-		Collector          *Collector       `yaml:"collector"           mapstructure:"collector"`
-		Watchers           *Watchers        `yaml:"watchers"            mapstructure:"watchers"`
-		SyslogServer       *SyslogServer    `yaml:"syslog_server"       mapstructure:"syslog_server"`
-		Labels             map[string]any   `yaml:"labels"              mapstructure:"labels"`
-		Version            string           `yaml:"-"`
-		Path               string           `yaml:"-"`
-		UUID               string           `yaml:"-"`
-		LibDir             string           `yaml:"-"`
-		AllowedDirectories []string         `yaml:"allowed_directories" mapstructure:"allowed_directories"`
-		Features           []string         `yaml:"features"            mapstructure:"features"`
+		Command            *Command            `yaml:"command"              mapstructure:"command"`
+		AuxiliaryCommand   *Command            `yaml:"auxiliary_command"    mapstructure:"auxiliary_command"`
+		Log                *Log                `yaml:"log"                  mapstructure:"log"`
+		DataPlaneConfig    *DataPlaneConfig    `yaml:"data_plane_config"    mapstructure:"data_plane_config"`
+		Client             *Client             `yaml:"client"               mapstructure:"client"`
+		Collector          *Collector          `yaml:"collector"            mapstructure:"collector"`
+		Watchers           *Watchers           `yaml:"watchers"             mapstructure:"watchers"`
+		ExternalDataSource *ExternalDataSource `yaml:"external_data_source" mapstructure:"external_data_source"`
+		SyslogServer       *SyslogServer       `yaml:"syslog_server"        mapstructure:"syslog_server"`
+		Labels             map[string]any      `yaml:"labels"               mapstructure:"labels"`
+		Version            string              `yaml:"-"`
+		Path               string              `yaml:"-"`
+		UUID               string              `yaml:"-"`
+		LibDir             string              `yaml:"-"`
+		AllowedDirectories []string            `yaml:"allowed_directories"  mapstructure:"allowed_directories"`
+		Features           []string            `yaml:"features"             mapstructure:"features"`
 	}
 
 	Log struct {
@@ -74,9 +75,10 @@ type (
 	}
 
 	Client struct {
-		HTTP    *HTTP    `yaml:"http"    mapstructure:"http"`
-		Grpc    *GRPC    `yaml:"grpc"    mapstructure:"grpc"`
-		Backoff *BackOff `yaml:"backoff" mapstructure:"backoff"`
+		HTTP                *HTTP         `yaml:"http"                  mapstructure:"http"`
+		Grpc                *GRPC         `yaml:"grpc"                  mapstructure:"grpc"`
+		Backoff             *BackOff      `yaml:"backoff"               mapstructure:"backoff"`
+		FileDownloadTimeout time.Duration `yaml:"file_download_timeout" mapstructure:"file_download_timeout"`
 	}
 
 	HTTP struct {
@@ -360,6 +362,17 @@ type (
 		Token      string        `yaml:"token,omitempty"       mapstructure:"token"`
 		Timeout    time.Duration `yaml:"timeout"               mapstructure:"timeout"`
 	}
+
+	ProxyURL struct {
+		URL string `yaml:"url" mapstructure:"url"`
+	}
+
+	ExternalDataSource struct {
+		ProxyURL         ProxyURL `yaml:"proxy"              mapstructure:"proxy"`
+		AllowedDomains   []string `yaml:"allowed_domains"    mapstructure:"allowed_domains"`
+		AllowedFileTypes []string `yaml:"allowed_file_types" mapstructure:"allowed_file_types"`
+		MaxBytes         int64    `yaml:"max_bytes"          mapstructure:"max_bytes"`
+	}
 )
 
 func (col *Collector) Validate(allowedDirectories []string) error {