⚡️ gcp: verify connection only once (#5442)

* ⚡️ gcp: verify connection only once

We are hitting rate limits like:
```
GCP integration error: rpc error: code = Unknown desc = googleapi: Error 429: Quota exceeded for quota metric 'Project V3 get requests' and limit 'Project V3 get requests per minute' of service 'cloudresourcemanager.googleapis.com' for consumer 'project_number:1234567890
```

This change reduces the number of requests by verifying the connection
only once per configuration. We have done this in the past for Github.

Closes #5095

Signed-off-by: Salim Afiune Maya <afiune@mondoo.com>

* ➡️  move connection options to diff struct

Because if not, we get different hashes all the time!

Signed-off-by: Salim Afiune Maya <afiune@mondoo.com>

---------

Signed-off-by: Salim Afiune Maya <afiune@mondoo.com>

Author: afiune

providers/gcp/connection/clients.go

@@ -22,11 +22,11 @@ func (c *GcpConnection) Credentials(scopes ...string) (*googleoauth.Credentials,
 	ctx := context.Background()
 	credParams := googleoauth.CredentialsParams{
 		Scopes:  scopes,
-		Subject: c.serviceAccountSubject,
+		Subject: c.opts.serviceAccountSubject,
 	}
-	if c.cred != nil {
+	if c.opts.cred != nil {
 		// use service account from secret
-		data, err := credsServiceAccountData(c.cred)
+		data, err := credsServiceAccountData(c.opts.cred)
 		if err != nil {
 			return nil, err
 		}
@@ -42,12 +42,12 @@ func (c *GcpConnection) Client(scope ...string) (*http.Client, error) {
 	ctx := context.Background()
 
 	// use service account from secret if one is provided
-	if c.cred != nil {
-		data, err := credsServiceAccountData(c.cred)
+	if c.opts.cred != nil {
+		data, err := credsServiceAccountData(c.opts.cred)
 		if err != nil {
 			return nil, err
 		}
-		return serviceAccountAuth(ctx, c.serviceAccountSubject, data, scope...)
+		return serviceAccountAuth(ctx, c.opts.serviceAccountSubject, data, scope...)
 	}
 
 	// otherwise fallback to default google sdk authentication

providers/gcp/connection/connection.go

@@ -6,6 +6,7 @@ package connection
 import (
 	"errors"
 
+	"github.com/mitchellh/hashstructure/v2"
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
@@ -33,6 +34,11 @@ type GcpConnection struct {
 	plugin.Connection
 	Conf  *inventory.Config
 	asset *inventory.Asset
+
+	opts gcpConnectionOptions
+}
+
+type gcpConnectionOptions struct {
 	// custom connection fields
 	resourceType ResourceType
 	resourceID   string
@@ -47,6 +53,7 @@ func NewGcpConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
 		Connection: plugin.NewConnection(id, asset),
 		Conf:       conf,
 		asset:      asset,
+		opts:       gcpConnectionOptions{},
 	}
 
 	// initialize connection
@@ -56,8 +63,11 @@ func NewGcpConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
 		cred = conf.Credentials[0]
 	}
 	if conf.Type == "gcp" {
-		if conf.Options == nil || (conf.Options["project-id"] == "" && conf.Options["organization-id"] == "" && conf.Options["folder-id"] == "") {
-			return nil, errors.New("google provider requires a gcp organization id, gcp project id or google workspace customer id. please set option `project-id` or `organization-id` or `customer-id` or `folder-id`")
+		if conf.Options == nil ||
+			(conf.Options["project-id"] == "" && conf.Options["organization-id"] == "" && conf.Options["folder-id"] == "") {
+			return nil, errors.New(
+				"google provider requires a gcp organization id, gcp project id or google workspace customer id. " +
+					"Please set option `project-id` or `organization-id` or `customer-id` or `folder-id`")
 		}
 	} else {
 		return nil, plugin.ErrProviderTypeDoesNotMatch
@@ -90,28 +100,45 @@ func NewGcpConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
 		override = conf.Options["platform-override"]
 	}
 
-	conn.resourceID = resourceID
-	conn.resourceType = resourceType
-	conn.cred = cred
-	conn.platformOverride = override
+	conn.opts.resourceID = resourceID
+	conn.opts.resourceType = resourceType
+	conn.opts.cred = cred
+	conn.opts.platformOverride = override
+
+	return conn, nil
+}
 
+func (c *GcpConnection) Hash() uint64 {
+	// generate hash of the config options used to generate this connection
+	// used to avoid verifying a client with the same options more than once
+	hash, err := hashstructure.Hash(c.opts, hashstructure.FormatV2, nil)
+	if err != nil {
+		log.Error().Err(err).Msg("unable to hash connection")
+	}
+	return hash
+}
+
+func (c *GcpConnection) Verify() error {
 	// verify that we have access to the organization or project
-	switch resourceType {
+	switch c.ResourceType() {
 	case Organization:
-		_, err := conn.GetOrganization(resourceID)
+		_, err := c.GetOrganization(c.ResourceID())
 		if err != nil {
-			log.Error().Err(err).Msgf("could not find or have no access to organization %s", resourceID)
-			return nil, err
+			log.Error().Err(err).
+				Str("organization", c.ResourceID()).
+				Msg("could not find, or have no access to organization")
+			return err
 		}
 	case Project, Gcr:
-		_, err := conn.GetProject(resourceID)
+		_, err := c.GetProject(c.ResourceID())
 		if err != nil {
-			log.Error().Err(err).Msgf("could not find or have no access to project %s", resourceID)
-			return nil, err
+			log.Error().Err(err).
+				Str("project", c.ResourceID()).
+				Msg("could not find, or have no access to project")
+			return err
 		}
 	}
-
-	return conn, nil
+	return nil
 }
 
 func (c *GcpConnection) Name() string {

providers/gcp/connection/gcp_organization.go

@@ -30,7 +30,7 @@ func (c *GcpConnection) OrganizationID() (string, error) {
 		}
 
 		// TODO: GetAncestry is not available in v3 anymore, we need to find an alternative approach
-		ancest, err := svc.Projects.GetAncestry(c.resourceID, &v1cloudresourcemanager.GetAncestryRequest{}).Do()
+		ancest, err := svc.Projects.GetAncestry(c.ResourceID(), &v1cloudresourcemanager.GetAncestryRequest{}).Do()
 		if err != nil {
 			return "", err
 		}
@@ -42,7 +42,7 @@ func (c *GcpConnection) OrganizationID() (string, error) {
 			}
 		}
 	case Organization:
-		return c.resourceID, nil
+		return c.ResourceID(), nil
 	}
 
 	return "", errors.New("could not find the organization")

providers/gcp/connection/gcpinstancesnapshot/provider.go

@@ -250,6 +250,16 @@ type GcpSnapshotConnection struct {
 	identifier      string
 }
 
+// no-op
+func (c *GcpSnapshotConnection) Hash() uint64 {
+	return 0
+}
+
+// no-op
+func (c *GcpSnapshotConnection) Verify() error {
+	return nil
+}
+
 func (c *GcpSnapshotConnection) Close() {
 	log.Debug().Msg("closing gcp snapshot connection")
 	if c == nil {

providers/gcp/connection/platform.go

@@ -29,37 +29,37 @@ func NewResourcePlatformID(service, project, region, objectType, name string) st
 func (c *GcpConnection) Identifier() (string, error) {
 	switch c.ResourceType() {
 	case Organization:
-		return NewOrganizationPlatformID(c.resourceID), nil
+		return NewOrganizationPlatformID(c.ResourceID()), nil
 	case Project:
-		return NewProjectPlatformID(c.resourceID), nil
+		return NewProjectPlatformID(c.ResourceID()), nil
 	case Folder:
-		return NewFolderPlatformID(c.resourceID), nil
+		return NewFolderPlatformID(c.ResourceID()), nil
 	default:
 		return "", fmt.Errorf("unsupported resource type %d", c.ResourceType())
 	}
 }
 
 func (c *GcpConnection) ResourceType() ResourceType {
-	return c.resourceType
+	return c.opts.resourceType
 }
 
 func (c *GcpConnection) ResourceID() string {
-	return c.resourceID
+	return c.opts.resourceID
 }
 
 func (c *GcpConnection) PlatformInfo() (*inventory.Platform, error) {
 	// TODO: this is a hack and we need to find a better way to do this
-	if c.platformOverride != "" && c.platformOverride != "gcp" {
+	if c.opts.platformOverride != "" && c.opts.platformOverride != "gcp" {
 		return &inventory.Platform{
-			Name:    c.platformOverride,
-			Title:   getTitleForPlatformName(c.platformOverride),
+			Name:    c.opts.platformOverride,
+			Title:   getTitleForPlatformName(c.opts.platformOverride),
 			Family:  []string{"google"},
 			Kind:    "gcp-object",
 			Runtime: "gcp",
 		}, nil
 	}
 
-	switch c.resourceType {
+	switch c.ResourceType() {
 	case Organization:
 		return &inventory.Platform{
 			Name:    "gcp-org",

providers/gcp/connection/shared/shared.go

@@ -16,6 +16,10 @@ type GcpConnection interface {
 	Type() ConnectionType
 	Config() *inventory.Config
 	Asset() *inventory.Asset
+
+	// Used to avoid verifying a client with the same options more than once
+	Verify() error
+	Hash() uint64
 }
 
 type Capabilities byte

providers/gcp/go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/cockroachdb/errors v1.11.3
 	github.com/google/go-containerregistry v0.20.3
 	github.com/hashicorp/go-cleanhttp v0.5.2
+	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/rs/zerolog v1.34.0
 	github.com/stretchr/testify v1.10.0
 	go.mondoo.com/cnquery/v11 v11.47.1
@@ -60,7 +61,6 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/knqyf263/go-rpmdb v0.1.1 // indirect
-	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/moby/buildkit v0.19.0 // indirect
 	github.com/moby/sys/mount v0.3.4 // indirect
 	github.com/moby/sys/mountinfo v0.7.2 // indirect
@@ -69,6 +69,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/package-url/packageurl-go v0.1.3 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/prometheus/client_golang v1.20.5 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect

providers/gcp/go.sum

@@ -723,6 +723,8 @@ github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoX
 github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=

providers/gcp/provider/provider.go

@@ -6,35 +6,43 @@ package provider
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
+	"time"
 
+	"github.com/rs/zerolog/log"
 	"go.mondoo.com/ranger-rpc/codes"
 	"go.mondoo.com/ranger-rpc/status"
 
 	"go.mondoo.com/cnquery/v11"
-	"go.mondoo.com/cnquery/v11/providers-sdk/v1/vault"
-
 	"go.mondoo.com/cnquery/v11/llx"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/upstream"
+	"go.mondoo.com/cnquery/v11/providers-sdk/v1/util/memoize"
+	"go.mondoo.com/cnquery/v11/providers-sdk/v1/vault"
 	"go.mondoo.com/cnquery/v11/providers/gcp/connection"
 	"go.mondoo.com/cnquery/v11/providers/gcp/connection/gcpinstancesnapshot"
 	"go.mondoo.com/cnquery/v11/providers/gcp/connection/shared"
 	"go.mondoo.com/cnquery/v11/providers/gcp/resources"
 )
 
-const (
-	ConnectionType = "gcp"
+const ConnectionType = "gcp"
+
+var (
+	cacheExpirationTime = 3 * time.Hour
+	cacheCleanupTime    = 6 * time.Hour
 )
 
 type Service struct {
 	*plugin.Service
+	*memoize.Memoizer
 }
 
 func Init() *Service {
 	return &Service{
-		Service: plugin.NewService(),
+		plugin.NewService(),
+		memoize.NewMemoizer(cacheExpirationTime, cacheCleanupTime),
 	}
 }
 
@@ -237,6 +245,15 @@ func (s *Service) connect(req *plugin.ConnectReq, callback plugin.ProviderCallba
 			return nil, err
 		}
 
+		// verify the connection only once
+		_, err, _ = s.Memoize(fmt.Sprintf("conn_%d", conn.Hash()), func() (interface{}, error) {
+			log.Trace().Str("type", string(conn.Type())).Msg("verifying connection client")
+			return nil, conn.Verify()
+		})
+		if err != nil {
+			return nil, err
+		}
+
 		var upstream *upstream.UpstreamClient
 		if req.Upstream != nil && !req.Upstream.Incognito {
 			upstream, err = req.Upstream.InitClient(context.Background())