⭐ Allow overriding provider registry url (#5943)

* ⭐ Allow overriding provider registry url

You can override the provider registry by either setting
```
registry_url: https://releases.mondoo.com/providers
```
in your mondoo config

or by setting the `MONDOO_REGISTRY_URL` environment variable

Future Improvements:
- We should cache the provider manifest for some period of time
- We can use etags to check if the provider manifest has actually
  changed

* rename registry_url to providers_url

* fix lint

* fix issue with the providers_url passed through --config not being used

* add providers url to status

* fix incorrect check

* fix lint

* fix(lint): handle errcheck by assigning viper.BindEnv results to _

Signed-off-by: Gary Bright <gary@mondoo.com>

* remove failing ci test

* don't keep retrying on econnrefused

* default providers registry url

* Fix early config parsing

---------

Signed-off-by: Gary Bright <gary@mondoo.com>
Co-authored-by: Gary Bright <gary@mondoo.com>

Author: jaym

apps/cnquery/cmd/root.go

@@ -120,7 +120,8 @@ func init() {
 	viper.BindPFlag("log-level", rootCmd.PersistentFlags().Lookup("log-level"))
 	viper.BindPFlag("api_proxy", rootCmd.PersistentFlags().Lookup("api-proxy"))
 	viper.BindPFlag("auto_update", rootCmd.PersistentFlags().Lookup("auto-update"))
-	viper.BindEnv("features")
+	_ = viper.BindEnv("features")
+	_ = viper.BindEnv("providers_url")
 
 	config.Init(rootCmd)
 }

apps/cnquery/cmd/status.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"os"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/cockroachdb/errors"
@@ -142,6 +143,12 @@ func checkStatus() (Status, error) {
 		}
 	}
 
+	if opts.ProvidersURL != "" {
+		s.Client.ProvidersURL = opts.ProvidersURL
+	} else {
+		s.Client.ProvidersURL = providers.DefaultProviderRegistryURL
+	}
+
 	return s, nil
 }
 
@@ -164,6 +171,7 @@ type ClientStatus struct {
 	Hostname       string              `json:"hostname,omitempty"`
 	Registered     bool                `json:"registered,omitempty"`
 	PingPongError  error               `json:"pingPongError,omitempty"`
+	ProvidersURL   string              `json:"providersUrl,omitempty"`
 }
 
 func (s Status) RenderCliStatus() {
@@ -188,6 +196,8 @@ func (s Status) RenderCliStatus() {
 		}
 	}
 
+	log.Info().Msg("Providers URL:\t" + s.Client.ProvidersURL)
+
 	installed, outdated, err := getProviders()
 	if err != nil {
 		log.Warn().Err(err).Msg("failed to get provider info")
@@ -260,16 +270,24 @@ func getProviders() ([]string, []string, error) {
 	if err != nil {
 		return nil, nil, err
 	}
+	var errCircuitBreaker error
 	for _, provider := range allProviders {
 		installed = append(installed, provider.Name)
-		latestVersion, err := providers.LatestVersion(provider.Name)
-		if err != nil {
-			continue
-		}
-		if latestVersion != provider.Version && provider.Name != "core" {
-			outdated = append(outdated, provider.Name)
+		if errCircuitBreaker == nil {
+			latestVersion, err := providers.LatestVersion(context.Background(), provider.Name)
+			if err != nil {
+				// If we get a connection refused, we assume this will happen for all providers
+				// so we will return early
+				if errors.Is(err, syscall.ECONNREFUSED) {
+					errCircuitBreaker = err
+				}
+				continue
+			}
+			if latestVersion != provider.Version && provider.Name != "core" {
+				outdated = append(outdated, provider.Name)
+			}
 		}
 	}
 
-	return installed, outdated, nil
+	return installed, outdated, errCircuitBreaker
 }

cli/config/config.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"go.mondoo.com/cnquery/v12"
 	"go.mondoo.com/cnquery/v12/logger"
@@ -49,6 +50,19 @@ func Init(rootCmd *cobra.Command) {
 	})
 	// persistent flags are global for the application
 	rootCmd.PersistentFlags().StringVar(&UserProvidedPath, "config", "", "Set config file path (default $HOME/.config/mondoo/mondoo.yml)")
+
+	// We need to parse the flags really early in the process, so that
+	// the config path is set before we initialize viper. This is because
+	// the providers configuration needs to be available before the rootCmd
+	// is executed as it does things like tries to download a provider if its missing
+	// See AttachCLIs in cli/providers/providers.go
+	flags := pflag.NewFlagSet("set", pflag.ContinueOnError)
+	flags.ParseErrorsAllowlist.UnknownFlags = true
+	flags.StringVar(&UserProvidedPath, "config", "", "")
+	flags.BoolP("help", "h", false, "")
+	if err := flags.Parse(os.Args); err != nil {
+		log.Debug().Err(err).Msg("could not parse flags")
+	}
 }
 
 func InitViperConfig() {
@@ -227,6 +241,11 @@ type CommonOpts struct {
 
 	// annotations that will be applied to all assets
 	Annotations map[string]string `json:"annotations,omitempty" mapstructure:"annotations"`
+
+	// ProvidersURL is the URL where providers are downloaded from
+	// if not set, the default Mondoo provider URL is used
+	// This can be a custom URL for an internal provider registry
+	ProvidersURL string `json:"providers_url,omitempty" mapstructure:"providers_url"`
 }
 
 // Workload Identity Federation

cli/providers/providers.go

@@ -53,10 +53,17 @@ func AttachCLIs(rootCmd *cobra.Command, commands ...*Command) error {
 	return nil
 }
 
-func detectConnectorName(args []string, rootCmd *cobra.Command, commands []*Command, providers providers.Providers) (string, bool) {
+func detectConnectorName(args []string, rootCmd *cobra.Command, commands []*Command, existing providers.Providers) (string, bool) {
 	autoUpdate := true
 
 	config.InitViperConfig()
+
+	if viper.IsSet("providers_url") {
+		if providersURL := viper.GetString("providers_url"); providersURL != "" {
+			providers.SetProviderRegistry(providers.NewMondooProviderRegistry(providers.WithBaseURL(providersURL)))
+		}
+	}
+
 	if viper.IsSet("auto_update") {
 		autoUpdate = viper.GetBool("auto_update")
 	}
@@ -84,8 +91,8 @@ func detectConnectorName(args []string, rootCmd *cobra.Command, commands []*Comm
 		attachPFlags(flags, cmd.Command.Flags())
 	}
 
-	for i := range providers {
-		provider := providers[i]
+	for i := range existing {
+		provider := existing[i]
 		for j := range provider.Connectors {
 			conn := provider.Connectors[j]
 			for k := range conn.Flags {

providers/providers.go

@@ -6,6 +6,7 @@ package providers
 import (
 	"archive/tar"
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
 	"net"
@@ -407,9 +408,10 @@ func EnsureProvider(search ProviderLookup, autoUpdate bool, existing Providers)
 }
 
 func Install(name string, version string) (*Provider, error) {
+	ctx := context.Background()
 	if version == "" {
 		// if no version is specified, we default to installing the latest one
-		latestVersion, err := LatestVersion(name)
+		latestVersion, err := registry.GetLatestVersion(ctx, name)
 		if err != nil {
 			return nil, err
 		}
@@ -419,65 +421,42 @@ func Install(name string, version string) (*Provider, error) {
 	log.Info().
 		Str("version", version).
 		Msg("installing provider '" + name + "'")
-	return installVersion(name, version)
+	return installVersion(ctx, name, version)
 }
 
-// This is the default installation source for core providers.
-const upstreamURL = "https://releases.mondoo.com/providers/{NAME}/{VERSION}/{NAME}_{VERSION}_{OS}_{ARCH}.tar.xz"
+func installVersion(ctx context.Context, name string, version string) (*Provider, error) {
+	logCtx := log.With().Str("provider", name).Str("version", version).Logger()
 
-func installVersion(name string, version string) (*Provider, error) {
-	url := upstreamURL
-	url = strings.ReplaceAll(url, "{NAME}", name)
-	url = strings.ReplaceAll(url, "{VERSION}", version)
-	url = strings.ReplaceAll(url, "{OS}", runtime.GOOS)
-	url = strings.ReplaceAll(url, "{ARCH}", runtime.GOARCH)
-
-	log.Debug().Str("url", url).Msg("installing provider from URL")
-	client, err := httpClientWithRetry()
+	res, err := registry.DownloadProvider(ctx, name, version, runtime.GOOS, runtime.GOARCH)
 	if err != nil {
 		return nil, err
 	}
-
-	res, err := client.Get(url)
-	if err != nil {
-		log.Debug().Str("url", url).Msg("failed to install from URL (get request)")
-		return nil, errors.Wrap(err, "failed to install "+name+"-"+version)
-	}
-
-	if res.StatusCode == http.StatusNotFound {
-		return nil, errors.New("cannot find provider " + name + "-" + version + " under url " + url)
-	} else if res.StatusCode != http.StatusOK {
-		log.Debug().Str("url", url).Int("status", res.StatusCode).Msg("failed to install from URL (status code)")
-		return nil, errors.New("failed to install " + name + "-" + version + ", received status code: " + res.Status)
-	}
-
-	log.Debug().Str("url", url).Msg("request complete, installing provider")
+	defer res.Close()
 
 	// we have to create new Reader to get rid of the timeouts imposed by the http client
 	var tar []byte
-	if tar, err = io.ReadAll(res.Body); err != nil {
-		log.Debug().Str("url", url).Msg("failed to install from URL (read body)")
+	if tar, err = io.ReadAll(res); err != nil {
+		logCtx.Debug().Msg("failed to read body of provider download")
 		return nil, errors.Wrap(err, "failed to install "+name+"-"+version+", failed to read body")
 	}
 
 	reader := io.NopCloser(
 		bytes.NewReader(tar),
 	)
 
-	// else we know we got a 200 response, we can safely install
 	installed, err := InstallIO(reader, InstallConf{
 		Dst: DefaultPath,
 	})
 	if err != nil {
-		log.Debug().Str("url", url).Msg("failed to install from URL (download)")
+		logCtx.Debug().Msg("failed to install provider")
 		return nil, errors.Wrap(err, "failed to install "+name+"-"+version)
 	}
 
 	if len(installed) == 0 {
 		return nil, errors.New("couldn't find installed provider")
 	}
 	if len(installed) > 1 {
-		log.Warn().Msg("too many providers were installed")
+		logCtx.Warn().Msg("too many providers were installed")
 	}
 	if installed[0].Version != version {
 		return nil, errors.New("version for provider didn't match expected install version: expected " + version + ", installed: " + installed[0].Version)
@@ -524,44 +503,6 @@ func installDependencies(provider *Provider, existing Providers) error {
 	return nil
 }
 
-func LatestVersion(name string) (string, error) {
-	client, err := httpClientWithRetry()
-	if err != nil {
-		return "", err
-	}
-
-	res, err := client.Get("https://releases.mondoo.com/providers/latest.json")
-	if err != nil {
-		return "", err
-	}
-
-	data, err := io.ReadAll(res.Body)
-	if err != nil {
-		log.Debug().Err(err).Msg("reading latest.json failed")
-		return "", errors.New("failed to read response from upstream provider versions")
-	}
-
-	var upstreamVersions ProviderVersions
-	err = json.Unmarshal(data, &upstreamVersions)
-	if err != nil {
-		log.Debug().Err(err).Msg("parsing latest.json failed")
-		return "", errors.New("failed to parse response from upstream provider versions")
-	}
-
-	var latestVersion string
-	for i := range upstreamVersions.Providers {
-		if upstreamVersions.Providers[i].Name == name {
-			latestVersion = upstreamVersions.Providers[i].Version
-			break
-		}
-	}
-
-	if latestVersion == "" {
-		return "", errors.New("cannot determine latest version of provider '" + name + "'")
-	}
-	return latestVersion, nil
-}
-
 func PrintInstallResults(providers []*Provider) {
 	for i := range providers {
 		provider := providers[i]
@@ -886,6 +827,7 @@ func readProviderDir(pdir string) (*Provider, error) {
 }
 
 func TryProviderUpdate(provider *Provider, update UpdateProvidersConfig) (*Provider, error) {
+	ctx := context.Background()
 	if provider.Path == "" {
 		return nil, errors.New("cannot determine installation path for provider")
 	}
@@ -909,7 +851,7 @@ func TryProviderUpdate(provider *Provider, update UpdateProvidersConfig) (*Provi
 		}
 	}
 
-	latest, err := LatestVersion(provider.Name)
+	latest, err := registry.GetLatestVersion(ctx, provider.Name)
 	if err != nil {
 		log.Warn().Msg(err.Error())
 		// we can just continue with the existing provider, no need to error up,
@@ -937,7 +879,7 @@ func TryProviderUpdate(provider *Provider, update UpdateProvidersConfig) (*Provi
 		Str("installed", provider.Version).
 		Str("latest", latest).
 		Msg("found a new version for '" + provider.Name + "' provider")
-	provider, err = installVersion(provider.Name, latest)
+	provider, err = installVersion(ctx, provider.Name, latest)
 	if err != nil {
 		return nil, err
 	}