[generator] add recursive component discovery for GitHub repositories

generators/generator.go

@@ -14,7 +14,17 @@ const (
 	gitHub      = "github"
 )
 
+type GeneratorOptions struct {
+	Recursive  bool
+	MaxDepth   int
+	Extensions []string
+}
+
 func NewGenerator(registrant, url, packageName string) (models.PackageManager, error) {
+	return NewGeneratorWithOptions(registrant, url, packageName, GeneratorOptions{})
+}
+
+func NewGeneratorWithOptions(registrant, url, packageName string, opts GeneratorOptions) (models.PackageManager, error) {
 	registrant = utils.ReplaceSpacesAndConvertToLowercase(registrant)
 	switch registrant {
 	case artifactHub:
@@ -26,6 +36,9 @@ func NewGenerator(registrant, url, packageName string) (models.PackageManager, e
 		return github.GitHubPackageManager{
 			PackageName: packageName,
 			SourceURL:   url,
+			Recursive:   opts.Recursive,
+			MaxDepth:    opts.MaxDepth,
+			Extensions:  opts.Extensions,
 		}, nil
 	}
 	return nil, ErrUnsupportedRegistrant(fmt.Errorf("generator not implemented for the registrant %s", registrant))

generators/generator_test.go

@@ -0,0 +1,64 @@
+package generators
+
+import (
+	"testing"
+
+	"github.com/meshery/meshkit/generators/github"
+)
+
+func TestNewGeneratorWithOptions(t *testing.T) {
+	tests := []struct {
+		name        string
+		registrant  string
+		url         string
+		packageName string
+		opts        GeneratorOptions
+		wantRec     bool
+		wantDepth   int
+	}{
+		{
+			name:        "Github Recursive",
+			registrant:  "github",
+			url:         "https://github.com/owner/repo",
+			packageName: "test",
+			opts: GeneratorOptions{
+				Recursive: true,
+				MaxDepth:  5,
+			},
+			wantRec:   true,
+			wantDepth: 5,
+		},
+		{
+			name:        "Github Default",
+			registrant:  "github",
+			url:         "https://github.com/owner/repo",
+			packageName: "test",
+			opts: GeneratorOptions{
+				Recursive: false,
+				MaxDepth:  0,
+			},
+			wantRec:   false,
+			wantDepth: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pm, err := NewGeneratorWithOptions(tt.registrant, tt.url, tt.packageName, tt.opts)
+			if err != nil {
+				t.Fatalf("NewGeneratorWithOptions() error = %v", err)
+			}
+
+			if ghpm, ok := pm.(github.GitHubPackageManager); ok {
+				if ghpm.Recursive != tt.wantRec {
+					t.Errorf("NewGeneratorWithOptions() Recursive = %v, want %v", ghpm.Recursive, tt.wantRec)
+				}
+				if ghpm.MaxDepth != tt.wantDepth {
+					t.Errorf("NewGeneratorWithOptions() MaxDepth = %v, want %v", ghpm.MaxDepth, tt.wantDepth)
+				}
+			} else {
+				t.Errorf("NewGeneratorWithOptions() returned unexpected type")
+			}
+		})
+	}
+}

generators/github/git_repo.go

@@ -18,6 +18,9 @@ type GitRepo struct {
 	// <git://github.com/owner/repo/branch/versiontag/root(path to the directory/file)>
 	URL         *url.URL
 	PackageName string
+	Recursive   bool
+	MaxDepth    int
+	Extensions  []string
 }
 
 // Assumpations: 1. Always a K8s manifest
@@ -54,10 +57,23 @@ func (gr GitRepo) GetContent() (models.Package, error) {
 		Owner(owner).
 		Repo(repo).
 		Branch(branch).
-		Root(root).
+		MaxDepth(gr.MaxDepth).
+		AllowedExtensions(gr.Extensions).
 		RegisterFileInterceptor(fileInterceptor(br)).
 		RegisterDirInterceptor(dirInterceptor(br))
 
+	effectiveRoot := root
+	if gr.Recursive {
+		if !strings.HasSuffix(effectiveRoot, "/**") {
+			effectiveRoot += "/**"
+		}
+	} else {
+		if strings.HasSuffix(effectiveRoot, "/**") {
+			effectiveRoot = strings.TrimSuffix(effectiveRoot, "/**")
+		}
+	}
+	gw = gw.Root(effectiveRoot)
+
 	if version != "" {
 		gw = gw.ReferenceName(fmt.Sprintf("refs/tags/%s", version))
 	}

generators/github/package_manager.go

@@ -11,6 +11,9 @@ import (
 type GitHubPackageManager struct {
 	PackageName string
 	SourceURL   string
+	Recursive   bool
+	MaxDepth    int
+	Extensions  []string
 }
 
 func (ghpm GitHubPackageManager) GetPackage() (models.Package, error) {
@@ -21,7 +24,7 @@ func (ghpm GitHubPackageManager) GetPackage() (models.Package, error) {
 	}
 	protocol := url.Scheme
 
-	downloader := NewDownloaderForScheme(protocol, url, ghpm.PackageName)
+	downloader := NewDownloaderForScheme(protocol, url, ghpm)
 	if downloader == nil {
 		err = errors.New("unsupported protocol")
 		return nil, ErrGenerateGitHubPackage(err, ghpm.PackageName)

generators/github/recursive_test.go

@@ -0,0 +1,186 @@
+package github
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/meshery/meshkit/utils/walker"
+)
+
+func TestRecursiveWalkFunctional(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "test-recursive-walk")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	r, err := git.PlainInit(tempDir, false)
+	if err != nil {
+		t.Fatalf("Failed to init git repo: %v", err)
+	}
+	w, err := r.Worktree()
+	if err != nil {
+		t.Fatalf("Failed to get worktree: %v", err)
+	}
+
+	createFile(t, tempDir, "root.yaml", "content")
+	createFile(t, tempDir, "root.json", "content")
+	createFile(t, tempDir, "level1/level1.yaml", "content")
+	createFile(t, tempDir, "level1/level1.txt", "content")
+	createFile(t, tempDir, "level1/level2/level2.yaml", "content")
+
+	_, err = w.Add(".")
+	if err != nil {
+		t.Fatalf("Failed to add files: %v", err)
+	}
+	_, err = w.Commit("Initial commit", &git.CommitOptions{
+		Author: &object.Signature{
+			Name:  "Test",
+			Email: "test@example.com",
+			When:  time.Now(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to commit: %v", err)
+	}
+
+	tests := []struct {
+		name       string
+		recursive  bool
+		maxDepth   int
+		extensions []string
+		wantFiles  []string
+	}{
+		{
+			name:       "Non-recursive (Root only, all types)",
+			recursive:  false,
+			maxDepth:   0,
+			extensions: nil,
+			wantFiles:  []string{"root.yaml", "root.json"},
+		},
+		{
+			name:       "Recursive Unlimited (All types)",
+			recursive:  true,
+			maxDepth:   0,
+			extensions: nil,
+			wantFiles:  []string{"root.yaml", "root.json", "level1.yaml", "level1.txt", "level2.yaml"},
+		},
+		{
+			name:       "Recursive MaxDepth 1 (All types)",
+			recursive:  true,
+			maxDepth:   1,
+			extensions: nil,
+			wantFiles:  []string{"root.yaml", "root.json", "level1.yaml", "level1.txt"},
+		},
+		{
+			name:       "Recursive MaxDepth 2 (All types)",
+			recursive:  true,
+			maxDepth:   2,
+			extensions: nil,
+			wantFiles:  []string{"root.yaml", "root.json", "level1.yaml", "level1.txt", "level2.yaml"},
+		},
+		{
+			name:       "Recursive Filtered (.yaml only)",
+			recursive:  true,
+			maxDepth:   0,
+			extensions: []string{".yaml"},
+			wantFiles:  []string{"root.yaml", "level1.yaml", "level2.yaml"},
+		},
+		{
+			name:       "Recursive Filtered (.yaml, .json)",
+			recursive:  true,
+			maxDepth:   0,
+			extensions: []string{".yaml", ".json"},
+			wantFiles:  []string{"root.yaml", "root.json", "level1.yaml", "level2.yaml"},
+		},
+		{
+			name:       "Recursive MaxDepth 1 Filtered (.yaml)",
+			recursive:  true,
+			maxDepth:   1,
+			extensions: []string{".yaml"},
+			wantFiles:  []string{"root.yaml", "level1.yaml"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			repoPath := filepath.ToSlash(tempDir)
+			if !strings.HasPrefix(repoPath, "/") {
+				repoPath = "/" + repoPath
+			}
+			fileURL := "file://" + repoPath
+
+			walkerInst := walker.NewGit().
+				Owner("").
+				Repo("").
+				BaseURL(fileURL).
+				Branch("master").
+				Root("").
+				MaxDepth(tt.maxDepth).
+				AllowedExtensions(tt.extensions)
+
+			var collectedFiles []string
+			walkerInst.RegisterFileInterceptor(func(f walker.File) error {
+				collectedFiles = append(collectedFiles, f.Name)
+				return nil
+			})
+
+			if tt.recursive {
+				walkerInst.Root("/**")
+			} else {
+				walkerInst.Root("")
+			}
+
+			err := walkerInst.Walk()
+			if err != nil {
+				t.Fatalf("Walk failed: %v", err)
+			}
+
+			assertFilesEqual(t, collectedFiles, tt.wantFiles)
+		})
+	}
+}
+
+func assertFilesEqual(t *testing.T, got, want []string) {
+	gotMap := make(map[string]struct{})
+	for _, f := range got {
+		gotMap[f] = struct{}{}
+	}
+	failed := false
+	for _, f := range want {
+		if _, ok := gotMap[f]; !ok {
+			t.Errorf("missing expected file: %s", f)
+			failed = true
+		} else {
+			delete(gotMap, f)
+		}
+	}
+	if len(gotMap) > 0 {
+		for f := range gotMap {
+			t.Errorf("unexpected file collected: %s", f)
+			failed = true
+		}
+	}
+	if failed {
+		t.Errorf("Got: %v", got)
+		t.Errorf("Want: %v", want)
+	}
+}
+
+func createFile(t *testing.T, base, path, content string) {
+	fullPath := filepath.Join(base, path)
+	err := os.MkdirAll(filepath.Dir(fullPath), 0755)
+	if err != nil {
+		t.Fatalf("Failed to create dirs: %v", err)
+	}
+	err = ioutil.WriteFile(fullPath, []byte(content), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create file: %v", err)
+	}
+}

generators/github/scheme_interface.go

@@ -10,19 +10,22 @@ type DownloaderScheme interface {
 	GetContent() (models.Package, error)
 }
 
-func NewDownloaderForScheme(scheme string, url *url.URL, packageName string) DownloaderScheme {
+func NewDownloaderForScheme(scheme string, url *url.URL, gp GitHubPackageManager) DownloaderScheme {
 	switch scheme {
 	case "git":
 		return GitRepo{
 			URL:         url,
-			PackageName: packageName,
+			PackageName: gp.PackageName,
+			Recursive:   gp.Recursive,
+			MaxDepth:    gp.MaxDepth,
+			Extensions:  gp.Extensions,
 		}
 	case "http":
 		fallthrough
 	case "https":
 		return URL{
 			URL:         url,
-			PackageName: packageName,
+			PackageName: gp.PackageName,
 		}
 	}
 	return nil