feat:[AH-2554]: Support for allowed blocked patterns per registry mapping in CLI (#148)

* 19bbf2 Added support for allowed blocked patterns per registry mapping

Author: cashwinichourasia

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/BurntSushi/toml v1.5.0
 	github.com/MakeNowJust/heredoc v1.0.0
 	github.com/getkin/kin-openapi v0.132.0
+	github.com/gobwas/glob v0.2.3
 	github.com/google/go-containerregistry v0.20.6
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-retryablehttp v0.7.8

go.sum

@@ -116,6 +116,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
 github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=

migrate_ex_config.yaml

@@ -29,6 +29,9 @@ mappings:
     destinationRegistry: dst
   - artifactType: MAVEN
     sourceRegistry: src
+    excludePatterns:
+      - com/*
+      - foo/*
     destinationRegistry: dst
   - artifactType: GENERIC
     sourceRegistry: src
@@ -47,4 +50,9 @@ mappings:
     destinationRegistry: dst
   - artifactType: COMPOSER
     sourceRegistry: src
-    destinationRegistry: dst
+    destinationRegistry: dst
+  - artifactType: DART
+    sourceRegistry: src
+    destinationRegistry: dst
+    includePatterns:
+      - packages/sample_dart_pkg/**

module/ar/migrate/migratable/registry.go

@@ -5,12 +5,12 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/harness/harness-cli/module/ar/migrate/adapter"
 	"github.com/harness/harness-cli/module/ar/migrate/engine"
 	"github.com/harness/harness-cli/module/ar/migrate/tree"
 	"github.com/harness/harness-cli/module/ar/migrate/types"
-
-	"github.com/google/uuid"
+	"github.com/harness/harness-cli/module/ar/migrate/util"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 )
@@ -81,7 +81,6 @@ func (r *Registry) Pre(ctx context.Context) error {
 	logger.Info().Msg("Starting registry pre-migration step")
 
 	startTime := time.Now()
-
 	registry, err := r.destAdapter.GetRegistry(ctx, r.destRegistry)
 	if err != nil {
 		log.Error().Err(err).Msgf("Failed to get registry %q", r.destRegistry)
@@ -107,13 +106,31 @@ func (r *Registry) Migrate(ctx context.Context) error {
 
 	logger.Info().Msg("Starting registry migration step")
 
+	if len(r.mapping.IncludePatterns) > 0 && len(r.mapping.ExcludePatterns) > 0 {
+		logger.Error().Msgf("Either include or Exclude Pattern is suppoted at a time for %s", r.artifactType)
+		return fmt.Errorf("failed in validating config file for %s ", r.artifactType)
+	}
+
 	startTime := time.Now()
 
 	files, err2 := r.srcAdapter.GetFiles(r.srcRegistry)
 	if err2 != nil {
 		logger.Error().Msgf("Failed to get files from registry %s", r.srcRegistry)
 		return fmt.Errorf("get files from registry %s failed: %w", r.srcRegistry, err2)
 	}
+
+	// Filter files based on include/exclude patterns
+	currArtifactType := r.artifactType
+	if util.IsFileLevelFilterableArtifact(currArtifactType) {
+		if len(r.mapping.IncludePatterns) > 0 || len(r.mapping.ExcludePatterns) > 0 {
+			originalCount := len(files)
+			filteredFiles := util.FilterFilesByPatterns(files, r.mapping.IncludePatterns, r.mapping.ExcludePatterns)
+			files = filteredFiles
+			logger.Info().Msgf("Filtered files: %d -> %d (includePatterns: %v, excludePatterns: %v)",
+				originalCount, len(files), r.mapping.IncludePatterns, r.mapping.ExcludePatterns)
+		}
+	}
+
 	root := tree.TransformToTree(files)
 
 	pkgs, err := r.srcAdapter.GetPackages(r.srcRegistry, r.artifactType, root)
@@ -122,6 +139,17 @@ func (r *Registry) Migrate(ctx context.Context) error {
 		return fmt.Errorf("get packages failed: %w", err)
 	}
 
+	// applying package level filter
+	if util.IsPackageLevelFilterableArtifact(currArtifactType) {
+		if len(r.mapping.IncludePatterns) > 0 || len(r.mapping.ExcludePatterns) > 0 {
+			originalCount := len(pkgs)
+			filteredPackages := util.FilterFilesByPatternsPackageName(pkgs, r.mapping.IncludePatterns, r.mapping.ExcludePatterns)
+			pkgs = filteredPackages
+			logger.Info().Msgf("Filtered packages: %d -> %d (includePatterns: %v, excludePatterns: %v)",
+				originalCount, len(pkgs), r.mapping.IncludePatterns, r.mapping.ExcludePatterns)
+		}
+	}
+
 	var jobs []engine.Job
 	for _, pkg := range pkgs {
 		treeNode, err2 := tree.GetNodeForPath(root, pkg.Path)

module/ar/migrate/util/patternUtil.go

@@ -0,0 +1,139 @@
+package util
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/gobwas/glob"
+	"github.com/harness/harness-cli/module/ar/migrate/types"
+)
+
+/* Patterns support * and ** wildcards:
+- * matches files in the current directory only (single level)
+- ** matches files in all subdirectories recursively (multi-level)
+*/
+
+func MatchesPattern(filePath string, patterns []string) bool {
+	if len(patterns) == 0 {
+		return true
+	}
+
+	// removing leading slashes
+	normalizedPath := strings.TrimPrefix(filePath, "/")
+
+	for _, pattern := range patterns {
+		normalizedPattern := strings.TrimPrefix(pattern, "/")
+
+		// Validate pattern - only * and ** wildcards are supported
+		if containsUnsupportedWildcards(normalizedPattern) {
+			fmt.Printf("WARNING: Pattern '%s' contains unsupported wildcard characters. Only * and ** are supported.\n", pattern)
+			continue
+		}
+
+		// The library handles *  and **  natively
+		g, err := glob.Compile(normalizedPattern, '/')
+		if err != nil {
+			// If pattern compilation fails, skip this pattern
+			continue
+		}
+
+		// Check if the path matches the pattern
+		if g.Match(normalizedPath) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// containsUnsupportedWildcards checks if pattern contains unsupported wildcard characters
+// Only * and ** are supported. Characters like ?, [, ], {, } are not supported.
+func containsUnsupportedWildcards(pattern string) bool {
+	unsupportedChars := []rune{'?', '[', ']', '{', '}'}
+
+	for _, char := range unsupportedChars {
+		if strings.ContainsRune(pattern, char) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// FilterFilesByPatterns filters a list of files based on include and exclude patterns.
+// Include patterns are applied first (if any), then exclude patterns are applied.
+// If no include patterns are specified, all files are included by default.
+
+func FilterFilesByPatterns(files []types.File, includePatterns, excludePatterns []string) []types.File {
+	if len(includePatterns) == 0 && len(excludePatterns) == 0 {
+		return files
+	}
+
+	var filtered []types.File
+	for _, file := range files {
+		// Skipping folders
+		if file.Folder {
+			filtered = append(filtered, file)
+			continue
+		}
+
+		if len(includePatterns) > 0 {
+			if !MatchesPattern(file.Uri, includePatterns) {
+				// File doesn't match any include pattern, skip it
+				continue
+			}
+		} else if len(excludePatterns) > 0 {
+			if MatchesPattern(file.Uri, excludePatterns) {
+				// File matches an exclude pattern, skip it
+				continue
+			}
+		}
+		//appending passed files
+		filtered = append(filtered, file)
+	}
+
+	return filtered
+}
+
+// This is to filter based on package name
+func FilterFilesByPatternsPackageName(packages []types.Package, includePatterns, excludePatterns []string) []types.Package {
+	if len(includePatterns) == 0 && len(excludePatterns) == 0 {
+		return packages
+	}
+
+	var filteredPackages []types.Package
+	for _, pkg := range packages {
+
+		if len(includePatterns) > 0 {
+			if !MatchesPattern(pkg.Name, includePatterns) {
+				continue
+			}
+		} else if len(excludePatterns) > 0 {
+			if MatchesPattern(pkg.Name, excludePatterns) {
+				continue
+			}
+		}
+		filteredPackages = append(filteredPackages, pkg)
+	}
+
+	return filteredPackages
+}
+
+func IsFileLevelFilterableArtifact(artifactType types.ArtifactType) bool {
+	switch artifactType {
+	case types.GENERIC, types.PYTHON, types.MAVEN, types.NUGET, types.NPM, types.DART, types.GO:
+		return true
+	default:
+		return false
+	}
+}
+
+func IsPackageLevelFilterableArtifact(artifactType types.ArtifactType) bool {
+
+	switch artifactType {
+	case types.DOCKER, types.HELM, types.HELM_LEGACY, types.RPM, types.CONDA, types.COMPOSER:
+		return true
+	default:
+		return false
+	}
+}