helper mode implementation for external file

go.mod

@@ -77,6 +77,7 @@ require (
 	go.opentelemetry.io/collector/scraper/scrapertest v0.135.0
 	go.opentelemetry.io/otel v1.38.0
 	go.uber.org/goleak v1.3.0
+	go.uber.org/mock v0.6.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/mod v0.28.0

go.sum

@@ -953,6 +953,8 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
+go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=

internal/config/config.go

@@ -122,6 +122,7 @@ func ResolveConfig() (*Config, error) {
 		Features:           viperInstance.GetStringSlice(FeaturesKey),
 		Labels:             resolveLabels(),
 		LibDir:             viperInstance.GetString(LibDirPathKey),
+		ExternalDataSource: resolveExternalDataSource(),
 	}
 
 	defaultCollector(collector, config)
@@ -426,6 +427,7 @@ func registerFlags() {
 	registerCollectorFlags(fs)
 	registerClientFlags(fs)
 	registerDataPlaneFlags(fs)
+	registerExternalDataSourceFlags(fs)
 
 	fs.SetNormalizeFunc(normalizeFunc)
 
@@ -440,6 +442,30 @@ func registerFlags() {
 	})
 }
 
+func registerExternalDataSourceFlags(fs *flag.FlagSet) {
+	fs.String(
+		ExternalDataSourceModeKey,
+		DefExternalDataSourceMode,
+		"Mode for external data source: 'direct' (HTTP/HTTPS) or 'helper'.",
+	)
+
+	fs.String(
+		ExternalDataSourceHelperPathKey,
+		DefExternalDataSourceHelperPath,
+		"Path to the helper executable for fetching external data sources.",
+	)
+	fs.StringSlice(
+		ExternalDataSourceAllowDomainsKey,
+		[]string{},
+		"List of allowed domains for external data sources.",
+	)
+	fs.Int64(
+		ExternalDataSourceMaxBytesKey,
+		DefExternalDataSourceMaxBytes,
+		"Maximum size in bytes for external data sources.",
+	)
+}
+
 func registerDataPlaneFlags(fs *flag.FlagSet) {
 	fs.Duration(
 		NginxReloadMonitoringPeriodKey,
@@ -1474,3 +1500,17 @@ func areCommandServerProxyTLSSettingsSet() bool {
 		viperInstance.IsSet(CommandServerProxyTLSSkipVerifyKey) ||
 		viperInstance.IsSet(CommandServerProxyTLSServerNameKey)
 }
+
+func resolveExternalDataSource() *ExternalDataSource {
+	externalDataSource := &ExternalDataSource{
+		Mode:           viperInstance.GetString(ExternalDataSourceModeKey),
+		AllowedDomains: viperInstance.GetStringSlice(ExternalDataSourceAllowDomainsKey),
+		MaxBytes:       viperInstance.GetInt64(ExternalDataSourceMaxBytesKey),
+	}
+
+	externalDataSource.Helper = &HelperConfig{
+		Path: viperInstance.GetString(ExternalDataSourceHelperPathKey),
+	}
+
+	return externalDataSource
+}

internal/config/config_test.go

@@ -1322,6 +1322,14 @@ func createConfig() *Config {
 			config.FeatureCertificates, config.FeatureFileWatcher, config.FeatureMetrics,
 			config.FeatureAPIAction, config.FeatureLogsNap,
 		},
+		ExternalDataSource: &ExternalDataSource{
+			Mode:           "",
+			AllowedDomains: nil,
+			Helper: &HelperConfig{
+				Path: "",
+			},
+			MaxBytes: 0,
+		},
 	}
 }
 

internal/config/defaults.go

@@ -111,6 +111,10 @@ const (
 
 	// File defaults
 	DefLibDir = "/var/lib/nginx-agent"
+
+	DefExternalDataSourceMode       = ""
+	DefExternalDataSourceHelperPath = ""
+	DefExternalDataSourceMaxBytes   = 100 * 1024 * 1024
 )
 
 func DefaultFeatures() []string {

internal/config/flags.go

@@ -25,6 +25,7 @@ const (
 	InstanceHealthWatcherMonitoringFrequencyKey = "watchers_instance_health_watcher_monitoring_frequency"
 	FileWatcherKey                              = "watchers_file_watcher"
 	LibDirPathKey                               = "lib_dir"
+	ExternalDataSourceRootKey                   = "external_data_source"
 )
 
 var (
@@ -137,6 +138,11 @@ var (
 
 	FileWatcherMonitoringFrequencyKey = pre(FileWatcherKey) + "monitoring_frequency"
 	NginxExcludeFilesKey              = pre(FileWatcherKey) + "exclude_files"
+
+	ExternalDataSourceModeKey         = pre(ExternalDataSourceRootKey) + "mode"
+	ExternalDataSourceHelperPathKey   = pre(ExternalDataSourceRootKey) + "helper_path"
+	ExternalDataSourceMaxBytesKey     = pre(ExternalDataSourceRootKey) + "max_bytes"
+	ExternalDataSourceAllowDomainsKey = pre(ExternalDataSourceRootKey) + "allowed_domains"
 )
 
 func pre(prefixes ...string) string {

internal/config/types.go

@@ -37,20 +37,21 @@ func parseServerType(str string) (ServerType, bool) {
 
 type (
 	Config struct {
-		Command            *Command         `yaml:"command"             mapstructure:"command"`
-		AuxiliaryCommand   *Command         `yaml:"auxiliary_command"   mapstructure:"auxiliary_command"`
-		Log                *Log             `yaml:"log"                 mapstructure:"log"`
-		DataPlaneConfig    *DataPlaneConfig `yaml:"data_plane_config"   mapstructure:"data_plane_config"`
-		Client             *Client          `yaml:"client"              mapstructure:"client"`
-		Collector          *Collector       `yaml:"collector"           mapstructure:"collector"`
-		Watchers           *Watchers        `yaml:"watchers"            mapstructure:"watchers"`
-		Labels             map[string]any   `yaml:"labels"              mapstructure:"labels"`
-		Version            string           `yaml:"-"`
-		Path               string           `yaml:"-"`
-		UUID               string           `yaml:"-"`
-		LibDir             string           `yaml:"-"`
-		AllowedDirectories []string         `yaml:"allowed_directories" mapstructure:"allowed_directories"`
-		Features           []string         `yaml:"features"            mapstructure:"features"`
+		Command            *Command            `yaml:"command"              mapstructure:"command"`
+		AuxiliaryCommand   *Command            `yaml:"auxiliary_command"    mapstructure:"auxiliary_command"`
+		ExternalDataSource *ExternalDataSource `yaml:"external_data_source" mapstructure:"external_data_source"`
+		Log                *Log                `yaml:"log"                  mapstructure:"log"`
+		DataPlaneConfig    *DataPlaneConfig    `yaml:"data_plane_config"    mapstructure:"data_plane_config"`
+		Client             *Client             `yaml:"client"               mapstructure:"client"`
+		Collector          *Collector          `yaml:"collector"            mapstructure:"collector"`
+		Watchers           *Watchers           `yaml:"watchers"             mapstructure:"watchers"`
+		Labels             map[string]any      `yaml:"labels"               mapstructure:"labels"`
+		Version            string              `yaml:"-"`
+		Path               string              `yaml:"-"`
+		UUID               string              `yaml:"-"`
+		LibDir             string              `yaml:"-"`
+		AllowedDirectories []string            `yaml:"allowed_directories"  mapstructure:"allowed_directories"`
+		Features           []string            `yaml:"features"             mapstructure:"features"`
 	}
 
 	Log struct {
@@ -350,6 +351,17 @@ type (
 		Token      string        `yaml:"token,omitempty"       mapstructure:"token"`
 		Timeout    time.Duration `yaml:"timeout"               mapstructure:"timeout"`
 	}
+
+	ExternalDataSource struct {
+		Helper         *HelperConfig `yaml:"helper"          mapstructure:"helper"`
+		Mode           string        `yaml:"mode"            mapstructure:"mode"`
+		AllowedDomains []string      `yaml:"allowed_domains" mapstructure:"allowed_domains"`
+		MaxBytes       int64         `yaml:"max_bytes"       mapstructure:"max_bytes"`
+	}
+
+	HelperConfig struct {
+		Path string `yaml:"path" mapstructure:"path"`
+	}
 )
 
 func (col *Collector) Validate(allowedDirectories []string) error {
@@ -479,6 +491,24 @@ func (c *Config) IsCommandServerProxyConfigured() bool {
 	return c.Command.Server.Proxy.URL != ""
 }
 
+func (c *Config) IsDomainAllowed(hostname string) bool {
+	allowedDomains := c.ExternalDataSource.AllowedDomains
+
+	for _, allowed := range allowedDomains {
+		// Handle wildcard domains like "*.vault.azure.com"
+		if strings.HasPrefix(allowed, "*.") {
+			suffix := strings.TrimPrefix(allowed, "*")
+			if strings.HasSuffix(hostname, suffix) {
+				return true
+			}
+		} else if hostname == allowed {
+			return true
+		}
+	}
+
+	return false
+}
+
 // isAllowedDir checks if the given path is in the list of allowed directories.
 // It returns true if the path is allowed, false otherwise.
 // If the path is allowed but does not exist, it also logs a warning.

internal/file/file_manager_service.go

@@ -12,9 +12,11 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"sync"
 
 	"google.golang.org/grpc"
@@ -56,6 +58,12 @@ type (
 		) (mpi.FileDataChunk_Content, error)
 		WriteManifestFile(ctx context.Context, updatedFiles map[string]*model.ManifestFile,
 			manifestDir, manifestPath string) (writeError error)
+		runHelper(
+			ctx context.Context,
+			helperPath string,
+			fileUrl string,
+			maxBytes int64,
+		) (string, error)
 		MoveFile(ctx context.Context, sourcePath, destPath string) error
 	}
 
@@ -156,6 +164,13 @@ func (fms *FileManagerService) ConfigApply(ctx context.Context,
 		return model.Error, allowedErr
 	}
 
+	isExternalFileReferenced := fms.isExternalFilePresent(fileOverview.GetFiles())
+	if isExternalFileReferenced {
+		if errExternalFileErr := fms.processExternalFiles(ctx, fileOverview.GetFiles()); errExternalFileErr != nil {
+			return model.Error, errExternalFileErr
+		}
+	}
+
 	diffFiles, fileContent, compareErr := fms.DetermineFileActions(
 		ctx,
 		fms.currentFilesOnDisk,
@@ -166,7 +181,7 @@ func (fms *FileManagerService) ConfigApply(ctx context.Context,
 		return model.Error, compareErr
 	}
 
-	if len(diffFiles) == 0 {
+	if len(diffFiles) == 0 && !isExternalFileReferenced {
 		return model.NoChange, nil
 	}
 
@@ -595,6 +610,37 @@ func (fms *FileManagerService) checkAllowedDirectory(checkFiles []*mpi.File) err
 	return nil
 }
 
+func (fms *FileManagerService) processExternalFiles(ctx context.Context, fileList []*mpi.File) error {
+	if helperAllowedErr := fms.checkHelperDirectory(); helperAllowedErr != nil {
+		return helperAllowedErr
+	}
+
+	return fms.downloadExternalFiles(ctx, fileList)
+}
+
+func (fms *FileManagerService) isExternalFilePresent(checkFiles []*mpi.File) bool {
+	for _, file := range checkFiles {
+		if file.GetExternalDataSource() != nil && file.GetExternalDataSource().GetLocation() != "" {
+			slog.Debug("External file source is present in file overview",
+				"location", file.GetExternalDataSource().GetLocation())
+
+			return true
+		}
+	}
+
+	return false
+}
+
+func (fms *FileManagerService) checkHelperDirectory() error {
+	allowed := fms.agentConfig.IsDirectoryAllowed(fms.agentConfig.ExternalDataSource.Helper.Path)
+	if !allowed {
+		return fmt.Errorf("helper file is not present in allowed directories %s",
+			fms.agentConfig.ExternalDataSource.Helper.Path)
+	}
+
+	return nil
+}
+
 func (fms *FileManagerService) convertToManifestFileMap(
 	currentFiles map[string]*mpi.File,
 	referenced bool,
@@ -669,3 +715,84 @@ func ConvertToMapOfFileCache(convertFiles []*mpi.File) map[string]*model.FileCac
 
 	return filesMap
 }
+
+func (fms *FileManagerService) downloadExternalFiles(ctx context.Context, files_list []*mpi.File) error {
+	downloadedFiles := make(map[string]string)
+
+	for _, file := range files_list {
+		if file.GetExternalDataSource() == nil {
+			continue
+		}
+		location := file.GetExternalDataSource().GetLocation()
+
+		if fms.agentConfig.ExternalDataSource.Mode != "helper" {
+			return fmt.Errorf("unsupported external data source mode: %s", fms.agentConfig.ExternalDataSource.Mode)
+		}
+
+		if parsedURL, err := url.Parse(location); err != nil {
+			return fmt.Errorf("invalid URL %s: %w", location, err)
+		} else if !fms.agentConfig.IsDomainAllowed(parsedURL.Hostname()) {
+			return fmt.Errorf("domain %s is not in the allowed list", parsedURL.Hostname())
+		}
+
+		if _, ok := downloadedFiles[location]; ok {
+			slog.DebugContext(ctx, "File already downloaded from external source", "location", location)
+			continue
+		}
+
+		if processErr := fms.processSingleExternalFile(ctx, file, location); processErr != nil {
+			return processErr
+		}
+
+		downloadedFiles[location] = file.GetFileMeta().GetName()
+	}
+
+	return nil
+}
+
+func (fms *FileManagerService) processSingleExternalFile(ctx context.Context, file *mpi.File, location string) error {
+	tmpFilePath, downloadErr := fms.fileOperator.runHelper(
+		ctx,
+		fms.agentConfig.ExternalDataSource.Helper.Path,
+		location,
+		fms.agentConfig.ExternalDataSource.MaxBytes,
+	)
+	if downloadErr != nil {
+		return fmt.Errorf("failed to download file from %s: %w", location, downloadErr)
+	}
+
+	content, readErr := os.ReadFile(tmpFilePath)
+	if readErr != nil {
+		os.Remove(tmpFilePath)
+		return fmt.Errorf("failed to read downloaded file from %s: %w", tmpFilePath, readErr)
+	}
+	os.Remove(tmpFilePath)
+
+	destPath := file.GetFileMeta().GetName()
+	destDir := filepath.Dir(destPath)
+
+	if _, statErr := os.Stat(destDir); os.IsNotExist(statErr) {
+		if mkdirErr := os.MkdirAll(destDir, os.ModePerm); mkdirErr != nil {
+			return fmt.Errorf("failed to create destination directory %s: %w", destDir, mkdirErr)
+		}
+	}
+
+	permission, parseErr := strconv.ParseUint(file.GetFileMeta().GetPermissions(), 8, 32)
+	if parseErr != nil {
+		slog.WarnContext(ctx, "failed to parse file permissions, using default 0644", "permissions",
+			file.GetFileMeta().GetPermissions())
+		permission = 0o644
+	}
+
+	if writeErr := os.WriteFile(destPath, content, os.FileMode(permission)); writeErr != nil {
+		return fmt.Errorf("failed to write content to final file %s: %w", destPath, writeErr)
+	}
+
+	file.FileMeta.Hash = files.GenerateHash(content)
+	file.FileMeta.Size = int64(len(content))
+	file.Unmanaged = true
+
+	slog.DebugContext(ctx, "Successfully downloaded file using helper", "location", location, "dest_path", destPath)
+
+	return nil
+}