third party config support

Author: Akshay2191

internal/config/config.go

@@ -123,6 +123,7 @@ func ResolveConfig() (*Config, error) {
 		Features:           viperInstance.GetStringSlice(FeaturesKey),
 		Labels:             resolveLabels(),
 		LibDir:             viperInstance.GetString(LibDirPathKey),
+		ExternalDataSource: resolveExternalDataSource(),
 	}
 
 	defaultCollector(collector, config)
@@ -434,6 +435,7 @@ func registerFlags() {
 	registerCollectorFlags(fs)
 	registerClientFlags(fs)
 	registerDataPlaneFlags(fs)
+	registerExternalDataSourceFlags(fs)
 
 	fs.SetNormalizeFunc(normalizeFunc)
 
@@ -448,6 +450,24 @@ func registerFlags() {
 	})
 }
 
+func registerExternalDataSourceFlags(fs *flag.FlagSet) {
+	fs.String(
+		ExternalDataSourceProxyUrlKey,
+		DefExternalDataSourceProxyUrl,
+		"Url to the proxy service to fetch the external file.",
+	)
+	fs.StringSlice(
+		ExternalDataSourceAllowDomainsKey,
+		[]string{},
+		"List of allowed domains for external data sources.",
+	)
+	fs.Int64(
+		ExternalDataSourceMaxBytesKey,
+		DefExternalDataSourceMaxBytes,
+		"Maximum size in bytes for external data sources.",
+	)
+}
+
 func registerDataPlaneFlags(fs *flag.FlagSet) {
 	fs.Duration(
 		NginxReloadMonitoringPeriodKey,
@@ -1506,3 +1526,26 @@ func areCommandServerProxyTLSSettingsSet() bool {
 		viperInstance.IsSet(CommandServerProxyTLSSkipVerifyKey) ||
 		viperInstance.IsSet(CommandServerProxyTLSServerNameKey)
 }
+
+func resolveExternalDataSource() *ExternalDataSource {
+	proxyURLStruct := ProxyURL{
+		URL: viperInstance.GetString(ExternalDataSourceProxyUrlKey),
+	}
+	externalDataSource := &ExternalDataSource{
+		ProxyURL:       proxyURLStruct,
+		AllowedDomains: viperInstance.GetStringSlice(ExternalDataSourceAllowDomainsKey),
+		MaxBytes:       viperInstance.GetInt64(ExternalDataSourceMaxBytesKey),
+	}
+
+	// Validate domains
+	if len(externalDataSource.AllowedDomains) > 0 {
+		for _, domain := range externalDataSource.AllowedDomains {
+			if strings.ContainsAny(domain, "/\\ ") || domain == "" {
+				slog.Error("domain is not specified in allowed_domains")
+				return nil
+			}
+		}
+	}
+
+	return externalDataSource
+}

internal/config/config_test.go

@@ -1325,6 +1325,13 @@ func createConfig() *Config {
 			config.FeatureCertificates, config.FeatureFileWatcher, config.FeatureMetrics,
 			config.FeatureAPIAction, config.FeatureLogsNap,
 		},
+		ExternalDataSource: &ExternalDataSource{
+			ProxyURL: ProxyURL{
+				URL: "",
+			},
+			AllowedDomains: nil,
+			MaxBytes:       0,
+		},
 	}
 }
 

internal/config/defaults.go

@@ -111,6 +111,9 @@ const (
 
 	// File defaults
 	DefLibDir = "/var/lib/nginx-agent"
+
+	DefExternalDataSourceProxyUrl = ""
+	DefExternalDataSourceMaxBytes = 100 * 1024 * 1024
 )
 
 func DefaultFeatures() []string {

internal/config/flags.go

@@ -25,6 +25,7 @@ const (
 	InstanceHealthWatcherMonitoringFrequencyKey = "watchers_instance_health_watcher_monitoring_frequency"
 	FileWatcherKey                              = "watchers_file_watcher"
 	LibDirPathKey                               = "lib_dir"
+	ExternalDataSourceRootKey                   = "external_data_source"
 )
 
 var (
@@ -138,6 +139,11 @@ var (
 
 	FileWatcherMonitoringFrequencyKey = pre(FileWatcherKey) + "monitoring_frequency"
 	NginxExcludeFilesKey              = pre(FileWatcherKey) + "exclude_files"
+
+	ExternalDataSourceProxyKey        = pre(ExternalDataSourceRootKey) + "proxy"
+	ExternalDataSourceProxyUrlKey     = pre(ExternalDataSourceProxyKey) + "url"
+	ExternalDataSourceMaxBytesKey     = pre(ExternalDataSourceRootKey) + "max_bytes"
+	ExternalDataSourceAllowDomainsKey = pre(ExternalDataSourceRootKey) + "allowed_domains"
 )
 
 func pre(prefixes ...string) string {

internal/config/types.go

@@ -36,20 +36,21 @@ func parseServerType(str string) (ServerType, bool) {
 
 type (
 	Config struct {
-		Command            *Command         `yaml:"command"             mapstructure:"command"`
-		AuxiliaryCommand   *Command         `yaml:"auxiliary_command"   mapstructure:"auxiliary_command"`
-		Log                *Log             `yaml:"log"                 mapstructure:"log"`
-		DataPlaneConfig    *DataPlaneConfig `yaml:"data_plane_config"   mapstructure:"data_plane_config"`
-		Client             *Client          `yaml:"client"              mapstructure:"client"`
-		Collector          *Collector       `yaml:"collector"           mapstructure:"collector"`
-		Watchers           *Watchers        `yaml:"watchers"            mapstructure:"watchers"`
-		Labels             map[string]any   `yaml:"labels"              mapstructure:"labels"`
-		Version            string           `yaml:"-"`
-		Path               string           `yaml:"-"`
-		UUID               string           `yaml:"-"`
-		LibDir             string           `yaml:"-"`
-		AllowedDirectories []string         `yaml:"allowed_directories" mapstructure:"allowed_directories"`
-		Features           []string         `yaml:"features"            mapstructure:"features"`
+		Watchers           *Watchers           `yaml:"watchers"             mapstructure:"watchers"`
+		Labels             map[string]any      `yaml:"labels"               mapstructure:"labels"`
+		Log                *Log                `yaml:"log"                  mapstructure:"log"`
+		DataPlaneConfig    *DataPlaneConfig    `yaml:"data_plane_config"    mapstructure:"data_plane_config"`
+		Client             *Client             `yaml:"client"               mapstructure:"client"`
+		Collector          *Collector          `yaml:"collector"            mapstructure:"collector"`
+		AuxiliaryCommand   *Command            `yaml:"auxiliary_command"    mapstructure:"auxiliary_command"`
+		ExternalDataSource *ExternalDataSource `yaml:"external_data_source" mapstructure:"external_data_source"`
+		Command            *Command            `yaml:"command"              mapstructure:"command"`
+		Path               string              `yaml:"-"`
+		Version            string              `yaml:"-"`
+		LibDir             string              `yaml:"-"`
+		UUID               string              `yaml:"-"`
+		AllowedDirectories []string            `yaml:"allowed_directories"  mapstructure:"allowed_directories"`
+		Features           []string            `yaml:"features"             mapstructure:"features"`
 	}
 
 	Log struct {
@@ -352,6 +353,16 @@ type (
 		Token      string        `yaml:"token,omitempty"       mapstructure:"token"`
 		Timeout    time.Duration `yaml:"timeout"               mapstructure:"timeout"`
 	}
+
+	ProxyURL struct {
+		URL string `yaml:"url" mapstructure:"url"`
+	}
+
+	ExternalDataSource struct {
+		ProxyURL       ProxyURL `yaml:"proxy"           mapstructure:"proxy"`
+		AllowedDomains []string `yaml:"allowed_domains" mapstructure:"allowed_domains"`
+		MaxBytes       int64    `yaml:"max_bytes"       mapstructure:"max_bytes"`
+	}
 )
 
 func (col *Collector) Validate(allowedDirectories []string) error {