feat: Support registering evaluation suites for multiple catalogs

pluginkit/evaluation_orchestrator.go

@@ -102,6 +102,22 @@ func (v *EvaluationOrchestrator) AddEvaluationSuite(catalogId string, loader Dat
 	return BAD_CATALOG(v.PluginName, fmt.Sprintf("no reference catalog found with id '%s'", catalogId), "aos40")
 }
 
+// AddEvaluationSuiteForAllCatalogs registers the provided steps for every
+// reference catalog that has been loaded via AddReferenceCatalogs.
+// This allows plugin developers to define their step implementations once and
+// have them automatically applied to all catalog versions.
+func (v *EvaluationOrchestrator) AddEvaluationSuiteForAllCatalogs(loader DataLoader, steps map[string][]gemara.AssessmentStep) error {
+	if len(v.referenceCatalogs) == 0 {
+		return BAD_CATALOG(v.PluginName, "no reference catalogs loaded", "aac10")
+	}
+	for catalogId := range v.referenceCatalogs {
+		if err := v.AddEvaluationSuite(catalogId, loader, steps); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (v *EvaluationOrchestrator) addEvaluationSuite(catalog *gemara.ControlCatalog, loader DataLoader, steps map[string][]gemara.AssessmentStep) {
 	importedControls := getImportedControls(catalog, v.referenceCatalogs)
 	catalog.Controls = append(catalog.Controls, importedControls...)

pluginkit/evaluation_orchestrator_test.go

@@ -116,6 +116,146 @@ func TestEvaluationOrchestrator_AddEvaluationSuite(t *testing.T) {
 	})
 }
 
+func TestEvaluationOrchestrator_AddEvaluationSuiteForAllCatalogs(t *testing.T) {
+	t.Run("Error Without Reference Catalogs", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		steps := map[string][]gemara.AssessmentStep{
+			"test-requirement": {step_Pass},
+		}
+
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(nil, steps)
+		if err == nil {
+			t.Error("Expected error when no reference catalogs are loaded")
+		}
+		if !strings.Contains(err.Error(), "no reference catalogs loaded") {
+			t.Errorf("Expected 'no reference catalogs loaded' error, got: %v", err)
+		}
+	})
+
+	t.Run("Error With Zero Controls Catalog", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		catalog := &gemara.ControlCatalog{
+			Metadata: gemara.Metadata{Id: "empty-catalog"},
+			Controls: []gemara.Control{},
+		}
+		orchestrator.referenceCatalogs = map[string]*gemara.ControlCatalog{
+			catalog.Metadata.Id: catalog,
+		}
+
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(nil, map[string][]gemara.AssessmentStep{})
+		if err == nil {
+			t.Error("Expected error for catalog with zero controls")
+		}
+		if !strings.Contains(err.Error(), "no controls provided") {
+			t.Errorf("Expected 'no controls provided' error, got: %v", err)
+		}
+	})
+
+	t.Run("Registers Suite For Each Catalog", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		catalog1 := getTestCatalogWithID("CCC.ObjStor")
+		catalog2 := getTestCatalogWithID("CCC.ObjStor-v2")
+		orchestrator.referenceCatalogs = map[string]*gemara.ControlCatalog{
+			catalog1.Metadata.Id: catalog1,
+			catalog2.Metadata.Id: catalog2,
+		}
+
+		steps := map[string][]gemara.AssessmentStep{
+			"CCC.Core.C01.TR01": {step_Pass},
+		}
+
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(nil, steps)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+
+		if len(orchestrator.possibleSuites) != 2 {
+			t.Errorf("Expected 2 suites (one per catalog), got %d",
+				len(orchestrator.possibleSuites))
+		}
+
+		// Verify each catalog got its own suite
+		suiteIDs := make(map[string]bool)
+		for _, suite := range orchestrator.possibleSuites {
+			suiteIDs[suite.CatalogId] = true
+		}
+		if !suiteIDs["CCC.ObjStor"] || !suiteIDs["CCC.ObjStor-v2"] {
+			t.Errorf("Expected suites for both catalogs, got: %v", suiteIDs)
+		}
+	})
+
+	t.Run("Uses Provided Loader", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		catalog := getTestCatalogWithRequirements()
+		orchestrator.referenceCatalogs = map[string]*gemara.ControlCatalog{
+			catalog.Metadata.Id: catalog,
+		}
+
+		testLoader := func(cfg *config.Config) (interface{}, error) {
+			return "suite-specific-payload", nil
+		}
+		steps := map[string][]gemara.AssessmentStep{
+			"CCC.Core.C01.TR01": {step_Pass},
+		}
+
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(testLoader, steps)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+
+		if len(orchestrator.possibleSuites) != 1 {
+			t.Errorf("Expected 1 suite for single catalog, got %d", len(orchestrator.possibleSuites))
+		}
+
+		for _, suite := range orchestrator.possibleSuites {
+			if suite.loader == nil {
+				t.Error("Expected suite loader to be set")
+			}
+		}
+	})
+
+	t.Run("Nil Steps", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		catalog := getTestCatalogWithRequirements()
+		orchestrator.referenceCatalogs = map[string]*gemara.ControlCatalog{
+			catalog.Metadata.Id: catalog,
+		}
+
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(nil, nil)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+
+		if len(orchestrator.possibleSuites) != 1 {
+			t.Errorf("Expected 1 suite, got %d", len(orchestrator.possibleSuites))
+		}
+		if orchestrator.possibleSuites[0].steps != nil {
+			t.Error("Expected nil steps to be preserved")
+		}
+	})
+
+	t.Run("Empty Steps", func(t *testing.T) {
+		orchestrator := &EvaluationOrchestrator{}
+		catalog := getTestCatalogWithRequirements()
+		orchestrator.referenceCatalogs = map[string]*gemara.ControlCatalog{
+			catalog.Metadata.Id: catalog,
+		}
+
+		emptySteps := map[string][]gemara.AssessmentStep{}
+		err := orchestrator.AddEvaluationSuiteForAllCatalogs(nil, emptySteps)
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+
+		if len(orchestrator.possibleSuites) != 1 {
+			t.Errorf("Expected 1 suite, got %d", len(orchestrator.possibleSuites))
+		}
+		if len(orchestrator.possibleSuites[0].steps) != 0 {
+			t.Error("Expected empty steps map to be preserved")
+		}
+	})
+}
+
 func TestEvaluationOrchestrator_Integration(t *testing.T) {
 	t.Run("Basic Setup", func(t *testing.T) {
 		orchestrator := &EvaluationOrchestrator{

pluginkit/test_data.go

@@ -36,19 +36,19 @@ func getTestCatalogWithNoRequirements() *gemara.ControlCatalog {
 	}
 }
 
-// getTestCatalogWithRequirements returns a catalog with controls and assessment requirements for tests that need a valid catalog.
-func getTestCatalogWithRequirements() *gemara.ControlCatalog {
+// getTestCatalogWithID returns a valid catalog with the given ID for tests that need multiple distinct catalogs.
+func getTestCatalogWithID(id string) *gemara.ControlCatalog {
 	return &gemara.ControlCatalog{
-		Metadata: gemara.Metadata{Id: "CCC.ObjStor"},
+		Metadata: gemara.Metadata{Id: id},
 		Controls: []gemara.Control{
 			{
 				Id:        "CCC.Core.C01",
 				Title:     "Encrypt Data for Transmission",
 				Objective: "Ensure that all communications are encrypted in transit.",
 				AssessmentRequirements: []gemara.AssessmentRequirement{
 					{
-						Id:           "CCC.Core.C01.TR01",
-						Text:         "When a port is exposed for non-SSH network traffic, all traffic MUST include a TLS handshake.",
+						Id:            "CCC.Core.C01.TR01",
+						Text:          "When a port is exposed for non-SSH network traffic, all traffic MUST include a TLS handshake.",
 						Applicability: requestedApplicability,
 					},
 				},
@@ -57,6 +57,11 @@ func getTestCatalogWithRequirements() *gemara.ControlCatalog {
 	}
 }
 
+// getTestCatalogWithRequirements returns a catalog with controls and assessment requirements for tests that need a valid catalog.
+func getTestCatalogWithRequirements() *gemara.ControlCatalog {
+	return getTestCatalogWithID("CCC.ObjStor")
+}
+
 func passingEvaluation() (evaluation *gemara.ControlEvaluation) {
 	evaluation = &gemara.ControlEvaluation{
 		Control: gemara.EntryMapping{