⭐️ add provider dependencies as builtin (#5503)

This change uses the new schema changes added at #5495 which exposes
dependencies that need to be consider in two scenarios:

1. Developing providers
2. Air-gapped environments

Now, when configuring a builtin provider, we will look at its
dependencies and add them to the providers configuration and build them
too.

Remote dependencies are not yet supported, we need to revisit this
change once we support them.

Signed-off-by: Salim Afiune Maya <afiune@mondoo.com>

Author: afiune

providers-sdk/v1/util/configure/configure.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"go/format"
+	"maps"
 	"os"
 	"os/exec"
 	"regexp"
@@ -19,6 +20,8 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"go.mondoo.com/cnquery/v11/logger"
+	cproviders "go.mondoo.com/cnquery/v11/providers"
+	"go.mondoo.com/cnquery/v11/providers-sdk/v1/resources"
 	"golang.org/x/mod/modfile"
 	"sigs.k8s.io/yaml"
 )
@@ -65,6 +68,13 @@ var rootCmd = &cobra.Command{
 			log.Fatal().Err(err).Str("path", confPath).Msg("failed to parse config file")
 		}
 
+		deps := buildProviders(conf.Builtin)
+		buildDependencies(deps)
+		for _, dep := range deps {
+			conf.AddProvider(dep)
+		}
+		log.Info().Strs("providers", conf.Providers()).Msg("(1/3) built providers")
+
 		builtinGo, err := genBuiltinGo(conf)
 		if err != nil {
 			log.Fatal().Err(err).Str("path", confPath).Msg("failed to generate builtin go")
@@ -73,10 +83,7 @@ var rootCmd = &cobra.Command{
 		if err = os.WriteFile(outPath, builtinGo, 0o644); err != nil {
 			log.Fatal().Err(err).Str("path", outPath).Msg("failed to write output")
 		}
-		log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(1/3) configured builtin providers")
-
-		buildProviders(conf.Builtin)
-		log.Info().Strs("providers", conf.Providers()).Msg("(2/3) built providers")
+		log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(2/3) configured builtin providers")
 
 		rewireDependencies(conf.Builtin)
 		log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(3/3) rewired dependencies/files")
@@ -208,7 +215,8 @@ func init() {
 }
 `
 
-func buildProviders(providers []Builtin) {
+func buildProviders(providers []Builtin) map[string]*resources.ProviderInfo {
+	deps := map[string]*resources.ProviderInfo{}
 	for i, provider := range providers {
 		cmd := exec.Command("make", "providers/build/"+provider.Name)
 		if provider.Remote != "" {
@@ -219,7 +227,9 @@ func buildProviders(providers []Builtin) {
 		var out bytes.Buffer
 		cmd.Stdout = &out
 		cmd.Stderr = &out
-		log.Debug().Str("provider", provider.Name).Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers)))
+		log.Debug().
+			Str("provider", provider.Name).
+			Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers)))
 		if err := cmd.Run(); err != nil {
 			fmt.Println(out.String())
 			log.Error().Err(err).Str("provider", provider.Name).Msg("failed to build provider")
@@ -237,6 +247,46 @@ func buildProviders(providers []Builtin) {
 		if err != nil {
 			log.Fatal().Err(err).Str("dst", dst).Msg("failed to write resources json")
 		}
+
+		// check for dependencies
+		schema := cproviders.MustLoadSchema(provider.Name, raw)
+		maps.Copy(deps, schema.Dependencies)
+	}
+	return deps
+}
+
+func buildDependencies(deps map[string]*resources.ProviderInfo) {
+	i := 0
+	for name, provider := range deps {
+		cmd := exec.Command("make", "providers/build/"+name)
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Stderr = &out
+		log.Debug().
+			Str("dependency", provider.Id).
+			Msg("build dependency " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(deps)))
+		if err := cmd.Run(); err != nil {
+			fmt.Println(out.String())
+			log.Error().Err(err).
+				Str("dependency", name).
+				Msg("failed to build provider dependency")
+		}
+
+		// Remote dependencies are not yet supported, we need to
+		// modify this once we support them.
+		// src := provider.Resource()
+		src := "providers/" + name + "/resources/" + name + ".resources.json"
+		raw, err := os.ReadFile(src)
+		if err != nil {
+			log.Fatal().Err(err).Str("src", src).Msg("failed to read resources json")
+		}
+
+		// dst := provider.Dist()
+		dst := "providers/" + name + ".resources.json"
+		err = os.WriteFile(dst, raw, 0o644)
+		if err != nil {
+			log.Fatal().Err(err).Str("dst", dst).Msg("failed to write resources json")
+		}
 	}
 }
 

providers-sdk/v1/util/configure/provider_conf.go

@@ -6,7 +6,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"slices"
 	"strings"
+
+	"go.mondoo.com/cnquery/v11/providers-sdk/v1/resources"
 )
 
 type ProvidersConf struct {
@@ -15,7 +18,26 @@ type ProvidersConf struct {
 	providers []string // Providers names
 }
 
-func (c ProvidersConf) Providers() []string {
+// AddProvider registers a new builtin provider into the providers configuration
+func (c *ProvidersConf) AddProvider(provider *resources.ProviderInfo) {
+	if slices.Contains(c.Providers(), provider.Name) {
+		return // provider already exist
+	}
+
+	// add provider
+	c.Builtin = append(c.Builtin, Builtin{
+		GoPackage: provider.Id,
+		Name:      provider.Name,
+		// Remote dependencies are not yet supported, we need to
+		// modify this once we support them.
+		// Remote:    provider.Remote,
+	})
+
+	// invalidate the internal list of providers since it changed
+	c.providers = nil
+}
+
+func (c *ProvidersConf) Providers() []string {
 	if len(c.providers) == 0 {
 		for _, b := range c.Builtin {
 			c.providers = append(c.providers, b.Name)