feat: add AddEvaluationSuiteForAllCatalogs to register suites for all loaded catalogs

Adds a new method to EvaluationOrchestrator that registers evaluation
suites for every reference catalog loaded via AddReferenceCatalogs.
This eliminates the need for plugins to re-parse catalog files to
extract IDs and loop over them individually.

Fixes #181

Author: Vinaya Damle

pluginkit/evaluation_orchestrator.go

@@ -102,6 +102,20 @@ func (v *EvaluationOrchestrator) AddEvaluationSuite(catalogId string, loader Dat
 	return BAD_CATALOG(v.PluginName, fmt.Sprintf("no reference catalog found with id '%s'", catalogId), "aos40")
 }
 
+// AddEvaluationSuiteForAllCatalogs registers the provided steps for every
+// reference catalog that has been loaded via AddReferenceCatalogs.
+// This allows plugin developers to define their step implementations once and
+// have them automatically applied to all catalog versions.
+func (v *EvaluationOrchestrator) AddEvaluationSuiteForAllCatalogs(loader DataLoader, steps map[string][]gemara.AssessmentStep) error {
+	if len(v.referenceCatalogs) == 0 {
+		return BAD_CATALOG(v.PluginName, "no reference catalogs loaded", "aac10")
+	}
+	for _, catalog := range v.referenceCatalogs {
+		v.addEvaluationSuite(catalog, loader, steps)
+	}
+	return nil
+}
+
 func (v *EvaluationOrchestrator) addEvaluationSuite(catalog *gemara.ControlCatalog, loader DataLoader, steps map[string][]gemara.AssessmentStep) {
 	importedControls := getImportedControls(catalog, v.referenceCatalogs)
 	catalog.Controls = append(catalog.Controls, importedControls...)

pluginkit/evaluation_orchestrator_test.go

@@ -116,6 +116,86 @@ func TestEvaluationOrchestrator_AddEvaluationSuite(t *testing.T) {
 	})
 }
 
+func TestEvaluationOrchestrator_AddEvaluationSuiteForAllCatalogs(t *testing.T) {
+	t.Run("Error Without Reference Catalogs", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		steps := map[string][]gemara.AssessmentStep{
+			"test-requirement": {step_Pass},
+		}
+
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(nil, steps)
+		if err == nil {
+			t.Error("Expected error when no reference catalogs are loaded")
+		}
+		if !strings.Contains(err.Error(), "no reference catalogs loaded") {
+			t.Errorf("Expected 'no reference catalogs loaded' error, got: %v", err)
+		}
+	})
+
+	t.Run("Registers Suite For Each Catalog", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		catalog1 := getTestCatalogWithID("CCC.ObjStor")
+		catalog2 := getTestCatalogWithID("CCC.ObjStor-v2")
+		orchestrator.referenceCatalogs = map[string]*gemara.ControlCatalog{
+			catalog1.Metadata.Id: catalog1,
+			catalog2.Metadata.Id: catalog2,
+		}
+
+		steps := map[string][]gemara.AssessmentStep{
+			"CCC.Core.C01.TR01": {step_Pass},
+		}
+
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(nil, steps)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+
+		if len(orchestrator.possibleSuites) != 2 {
+			t.Errorf("Expected 2 suites (one per catalog), got %d",
+				len(orchestrator.possibleSuites))
+		}
+
+		// Verify each catalog got its own suite
+		suiteIDs := make(map[string]bool)
+		for _, suite := range orchestrator.possibleSuites {
+			suiteIDs[suite.CatalogId] = true
+		}
+		if !suiteIDs["CCC.ObjStor"] || !suiteIDs["CCC.ObjStor-v2"] {
+			t.Errorf("Expected suites for both catalogs, got: %v", suiteIDs)
+		}
+	})
+
+	t.Run("Uses Provided Loader", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		catalog := getTestCatalogWithRequirements()
+		orchestrator.referenceCatalogs = map[string]*gemara.ControlCatalog{
+			catalog.Metadata.Id: catalog,
+		}
+
+		testLoader := func(cfg *config.Config) (interface{}, error) {
+			return "suite-specific-payload", nil
+		}
+		steps := map[string][]gemara.AssessmentStep{
+			"CCC.Core.C01.TR01": {step_Pass},
+		}
+
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(testLoader, steps)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+
+		if len(orchestrator.possibleSuites) != 1 {
+			t.Errorf("Expected 1 suite for single catalog, got %d", len(orchestrator.possibleSuites))
+		}
+
+		for _, suite := range orchestrator.possibleSuites {
+			if suite.loader == nil {
+				t.Error("Expected suite loader to be set")
+			}
+		}
+	})
+}
+
 func TestEvaluationOrchestrator_Integration(t *testing.T) {
 	t.Run("Basic Setup", func(t *testing.T) {
 		orchestrator := &EvaluationOrchestrator{

pluginkit/test_data.go

@@ -36,6 +36,13 @@ func getTestCatalogWithNoRequirements() *gemara.ControlCatalog {
 	}
 }
 
+// getTestCatalogWithID returns a valid catalog with the given ID for tests that need multiple distinct catalogs.
+func getTestCatalogWithID(id string) *gemara.ControlCatalog {
+	catalog := getTestCatalogWithRequirements()
+	catalog.Metadata.Id = id
+	return catalog
+}
+
 // getTestCatalogWithRequirements returns a catalog with controls and assessment requirements for tests that need a valid catalog.
 func getTestCatalogWithRequirements() *gemara.ControlCatalog {
 	return &gemara.ControlCatalog{