🐛 Enhance asset deduplication logic to evict subsets when supersets are added and update related tests

Author: slntopp

discovery/discovery.go

@@ -15,13 +15,13 @@ import (
 	"go.mondoo.com/mql/v13/cli/execruntime"
 	"go.mondoo.com/mql/v13/internal/workerpool"
 	"go.mondoo.com/mql/v13/llx"
-	"go.mondoo.com/mql/v13/utils/slicesx"
 	"go.mondoo.com/mql/v13/logger"
 	"go.mondoo.com/mql/v13/providers"
 	inventory "go.mondoo.com/mql/v13/providers-sdk/v1/inventory"
 	"go.mondoo.com/mql/v13/providers-sdk/v1/inventory/manager"
 	"go.mondoo.com/mql/v13/providers-sdk/v1/plugin"
 	"go.mondoo.com/mql/v13/providers-sdk/v1/upstream"
+	"go.mondoo.com/mql/v13/utils/slicesx"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -68,19 +68,30 @@ func (d *DiscoveredAssets) Add(asset *inventory.Asset, runtime *providers.Runtim
 		return false
 	}
 
+	// Evict existing assets whose IDs are a subset of the new asset's IDs.
+	// This ensures deduplication is order-independent.
+	kept := d.Assets[:0]
 	for _, existing := range d.Assets {
 		if slicesx.IsSubsetOf(asset.PlatformIds, existing.Asset.PlatformIds) {
 			log.Debug().Str("asset", asset.Name).Strs("platform-ids", asset.PlatformIds).Msg("discovery> skipping duplicate asset")
 			return false
 		}
+		if slicesx.IsSubsetOf(existing.Asset.PlatformIds, asset.PlatformIds) {
+			log.Debug().Str("asset", existing.Asset.Name).Strs("platform-ids", existing.Asset.PlatformIds).Msg("discovery> evicting asset that is a subset of new asset")
+			if existing.Runtime != nil {
+				existing.Runtime.Close()
+			}
+			continue
+		}
+		kept = append(kept, existing)
 	}
+	d.Assets = kept
 
 	log.Debug().Str("asset", asset.Name).Strs("platform-ids", asset.PlatformIds).Int("total", len(d.Assets)+1).Msg("discovery> added asset")
 	d.Assets = append(d.Assets, &AssetWithRuntime{Asset: asset, Runtime: runtime})
 	return true
 }
 
-
 func (d *DiscoveredAssets) AddError(asset *inventory.Asset, err error) {
 	d.assetsLock.Lock()
 	defer d.assetsLock.Unlock()

discovery/discovery_test.go

@@ -59,16 +59,16 @@ func TestDiscoveredAssetsAdd(t *testing.T) {
 		assert.Len(t, d.Assets, 1)
 	})
 
-	t.Run("subset rejection is order-independent", func(t *testing.T) {
+	t.Run("subset is evicted when superset arrives", func(t *testing.T) {
 		d := &DiscoveredAssets{}
 		fewer := &inventory.Asset{PlatformIds: []string{"hostname/node1", "ssh/ABC"}}
 		more := &inventory.Asset{PlatformIds: []string{"hostname/node1", "ssh/ABC", "cloud/xyz"}}
 
-		// Even when the subset is added first, the superset is still added
-		// (it has a new ID), and neither is incorrectly rejected.
+		// When the subset is added first, adding the superset evicts the subset.
 		assert.True(t, d.Add(fewer, nil))
 		assert.True(t, d.Add(more, nil))
-		assert.Len(t, d.Assets, 2)
+		assert.Len(t, d.Assets, 1)
+		assert.Equal(t, more.PlatformIds, d.Assets[0].Asset.PlatformIds)
 	})
 
 	t.Run("no cross-asset conflation", func(t *testing.T) {