Skip to content

[Issue - 10441] Improved k8s cluster discovery #5291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

[Issue - 10441] Improved k8s cluster discovery #5291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bca5ad5
Skipping discovery of Assets when filter is not provided and go into …
Chichichkin Mar 6, 2025
0ab56f6
refactor platformids creation for manifests and api
imilchev Mar 10, 2025
d367408
log meaningful error message when failing to list namespaces
imilchev Mar 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions providers/k8s/connection/admission/connection.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ func (c *Connection) InventoryConfig() *inventory.Config {
return c.asset.Connections[0]
}

func (c *Connection) BasePlatformId() (string, error) {
return c.AssetId()
}

func (c *Connection) AssetId() (string, error) {
reviews, err := c.AdmissionReviews()
if err != nil {
Expand Down
4 changes: 4 additions & 0 deletions providers/k8s/connection/api/connection.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ func (c *Connection) Platform() *inventory.Platform {
}
}

func (c *Connection) BasePlatformId() (string, error) {
return shared.IdPrefix, nil
}

func (c *Connection) AssetId() (string, error) {
// we use "kube-system" namespace uid as identifier for the cluster
// use the internal resources function to make sure we can get the right namespace
Expand Down
18 changes: 17 additions & 1 deletion providers/k8s/connection/manifest/connection.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,14 @@ func (c *Connection) Asset() *inventory.Asset {
return c.asset
}

func (c *Connection) BasePlatformId() (string, error) {
manifestHash, err := c.manifestHash()
if err != nil {
return "", err
}
return shared.IdPrefix + manifestHash, nil
}

func (c *Connection) AssetId() (string, error) {
// If we are doing an admission control scan, we have 1 resource in the manifest and it has a UID.
// Instead of using the file path to generate the ID, use the resource UID. We do this because for
Expand All @@ -168,6 +176,14 @@ func (c *Connection) AssetId() (string, error) {
}
}

manifestHash, err := c.manifestHash()
if err != nil {
return "", err
}
return shared.NewPlatformId(manifestHash), nil
}

func (c *Connection) manifestHash() (string, error) {
h := sha256.New()

// special handling for embedded content (e.g. piped in via stdin)
Expand All @@ -187,7 +203,7 @@ func (c *Connection) AssetId() (string, error) {
}

h.Write([]byte(absPath))
return shared.NewPlatformId(hex.EncodeToString(h.Sum(nil))), nil
return hex.EncodeToString(h.Sum(nil)), nil
}

func (c *Connection) InventoryConfig() *inventory.Config {
Expand Down
23 changes: 16 additions & 7 deletions providers/k8s/connection/manifest/connection_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func TestManifestDiscovery(t *testing.T) {
}
inv, err := resources.Discover(pluginRuntime, cnquery.Features{})
require.NoError(t, err)
require.Len(t, inv.Spec.Assets, 2)
require.Len(t, inv.Spec.Assets, 3)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to the comments in this PR, the manifest part didn't change. Why did this increase?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we discover the namespaces now. Before we skipped over them which was actually a bug. We turned on namespace discovery for api scans, but not for manifests


conn.InventoryConfig().Discover.Targets = []string{"all"}
pluginRuntime = &plugin.Runtime{
Expand All @@ -108,7 +108,7 @@ func TestManifestDiscovery(t *testing.T) {
}
inv, err = resources.Discover(pluginRuntime, cnquery.Features{})
require.NoError(t, err)
require.Len(t, inv.Spec.Assets, 2)
require.Len(t, inv.Spec.Assets, 3)

conn.InventoryConfig().Discover.Targets = []string{"deployments"}
pluginRuntime = &plugin.Runtime{
Expand Down Expand Up @@ -153,7 +153,7 @@ func TestOperatorManifest(t *testing.T) {
}
inv, err := resources.Discover(pluginRuntime, cnquery.Features{})
require.NoError(t, err)
require.Len(t, inv.Spec.Assets, 3)
require.Len(t, inv.Spec.Assets, 4)

require.Len(t, inv.Spec.Assets[1].PlatformIds, 1)

Expand All @@ -175,7 +175,8 @@ func TestOperatorManifest(t *testing.T) {

require.NotEqual(t, inv.Spec.Assets[0].PlatformIds[0], inv.Spec.Assets[1].PlatformIds[0])
require.Equal(t, "//platformid.api.mondoo.app/runtime/k8s/uid/"+manifestHash, inv.Spec.Assets[0].PlatformIds[0])
require.Equal(t, "//platformid.api.mondoo.app/runtime/k8s/uid/"+manifestHash+"/namespace/mondoo-operator/deployments/name/mondoo-operator-controller-manager", inv.Spec.Assets[2].PlatformIds[0])
require.Equal(t, "//platformid.api.mondoo.app/runtime/k8s/uid/"+manifestHash+"/namespace/mondoo-operator/services/name/mondoo-operator-controller-manager-metrics-service", inv.Spec.Assets[2].PlatformIds[0])
require.Equal(t, "//platformid.api.mondoo.app/runtime/k8s/uid/"+manifestHash+"/namespace/mondoo-operator/deployments/name/mondoo-operator-controller-manager", inv.Spec.Assets[3].PlatformIds[0])
}

func TestOperatorManifestWithNamespaceFilter(t *testing.T) {
Expand Down Expand Up @@ -222,9 +223,17 @@ func TestOperatorManifestWithNamespaceFilter(t *testing.T) {
require.NoError(t, err)
require.NotEmpty(t, asset.PlatformIds[0])
}

h := sha256.New()
absPath, err := filepath.Abs(path)
require.NoError(t, err)
h.Write([]byte(absPath))
manifestHash := hex.EncodeToString(h.Sum(nil))
require.NoError(t, err)

require.NotEqual(t, inv.Spec.Assets[0].PlatformIds[0], inv.Spec.Assets[1].PlatformIds[0])
require.Equal(t, "//platformid.api.mondoo.app/runtime/k8s/uid/namespace/mondoo-operator", inv.Spec.Assets[0].PlatformIds[0])
require.Equal(t, "//platformid.api.mondoo.app/runtime/k8s/uid/namespace/mondoo-operator/deployments/name/mondoo-operator-controller-manager", inv.Spec.Assets[2].PlatformIds[0])
require.Equal(t, "//platformid.api.mondoo.app/runtime/k8s/uid/"+manifestHash+"/namespace/mondoo-operator", inv.Spec.Assets[0].PlatformIds[0])
require.Equal(t, "//platformid.api.mondoo.app/runtime/k8s/uid/"+manifestHash+"/namespace/mondoo-operator/deployments/name/mondoo-operator-controller-manager", inv.Spec.Assets[2].PlatformIds[0])
}

func TestManifestNoObjects(t *testing.T) {
Expand Down Expand Up @@ -316,5 +325,5 @@ func TestManifestDir(t *testing.T) {
}
require.NotEmpty(t, inv.Spec.Assets[0].PlatformIds[0])
// we have the operator deployment twice
require.Equal(t, inv.Spec.Assets[1].PlatformIds[0], inv.Spec.Assets[2].PlatformIds[0])
require.Equal(t, inv.Spec.Assets[3].PlatformIds[0], inv.Spec.Assets[4].PlatformIds[0])
}
16 changes: 0 additions & 16 deletions providers/k8s/connection/manifest/testdata/no-discovered-objects.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,19 +319,3 @@ kind: ConfigMap
metadata:
name: mondoo-operator-manager-config
namespace: mondoo-operator
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/name: mondoo-operator
name: mondoo-operator-controller-manager-metrics-service
namespace: mondoo-operator
spec:
ports:
- name: metrics
port: 8080
protocol: TCP
targetPort: metrics
selector:
app.kubernetes.io/name: mondoo-operator
17 changes: 7 additions & 10 deletions providers/k8s/connection/shared/connection.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const (
OPTION_OBJECT_KIND = "object-kind"
OPTION_CONTEXT = "context"
OPTION_KUBELOGIN = "kubelogin"
idPrefix = "//platformid.api.mondoo.app/runtime/k8s/uid/"
IdPrefix = "//platformid.api.mondoo.app/runtime/k8s/uid/"
)

type ConnectionType string
Expand All @@ -44,6 +44,7 @@ type Connection interface {
Platform() *inventory.Platform
Asset() *inventory.Asset
AssetId() (string, error)
BasePlatformId() (string, error)

AdmissionReviews() ([]admissionv1.AdmissionReview, error)
Namespace(name string) (*v1.Namespace, error)
Expand Down Expand Up @@ -76,12 +77,12 @@ func sliceToPtrSlice[T any](items []T) []*T {
}

func NewPlatformId(assetId string) string {
return idPrefix + assetId
return IdPrefix + assetId
}

func NewWorkloadPlatformId(clusterIdentifier, workloadType, namespace, name, uid string) string {
func NewWorkloadPlatformId(basePlatformId, clusterIdentifier, workloadType, namespace, name, uid string) string {
if workloadType == "namespace" {
return NewNamespacePlatformId(clusterIdentifier, name, uid)
return NewNamespacePlatformId(basePlatformId, name, uid)
}

platformIdentifier := clusterIdentifier
Expand All @@ -95,10 +96,6 @@ func NewWorkloadPlatformId(clusterIdentifier, workloadType, namespace, name, uid
return platformIdentifier
}

func NewNamespacePlatformId(clusterIdentifier, name, uid string) string {
if clusterIdentifier == "" {
return fmt.Sprintf("%snamespace/%s", idPrefix, name)
}

return fmt.Sprintf("%s/namespace/%s/uid/%s", clusterIdentifier, name, uid)
func NewNamespacePlatformId(basePlatformId, name, uid string) string {
return fmt.Sprintf("%s%s/namespace/%s", basePlatformId, uid, name)
}
3 changes: 3 additions & 0 deletions providers/k8s/connection/shared/manifest_parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ func (t *ManifestParser) Namespaces() ([]v1.Namespace, error) {
o, err := meta.Accessor(res)
if err == nil {
ns := o.GetNamespace()
if ns == "" {
continue
}
// There are types of resources that do not have meta data. Instead of erroring
// skip them.
namespaceMap[ns] = struct{}{}
Expand Down
4 changes: 2 additions & 2 deletions providers/k8s/provider/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error)
return &res, nil
}

func (s *Service) MockConnect(req *plugin.ConnectReq, callback plugin.ProviderCallback) (*plugin.ConnectRes, error) {
func (s *Service) MockConnect(_ *plugin.ConnectReq, _ plugin.ProviderCallback) (*plugin.ConnectRes, error) {
return nil, errors.New("mock connect not yet implemented")
}

Expand All @@ -135,7 +135,7 @@ func (s *Service) Connect(req *plugin.ConnectReq, callback plugin.ProviderCallba
}

return &plugin.ConnectRes{
Id: uint32(conn.ID()),
Id: conn.ID(),
Name: conn.Name(),
Asset: req.Asset,
Inventory: inventory,
Expand Down
Loading
Loading