Skip to content

Commit 291ffe5

Browse files
nachandrnachandr
committed
Enhanced MTA-831 test to validate crane validate report in YAML format
Signed-off-by: Nandini Chandra <nachandr@redhat.com>
1 parent cd472ab commit 291ffe5

3 files changed

Lines changed: 207 additions & 103 deletions

File tree

‎e2e-tests/tests/tier0/mta_831_validate_compatible_resources_offline_test.go‎

Lines changed: 37 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -13,108 +13,8 @@ import (
1313
cranevalidate "github.com/konveyor/crane/internal/validate"
1414
. "github.com/onsi/ginkgo/v2"
1515
. "github.com/onsi/gomega"
16-
"gopkg.in/yaml.v3"
1716
)
1817

19-
func runValidateAndParseReport(runner CraneRunner, inputDir string, validateDir string,
20-
apiSurfaceFile string, outputFormat string) (cranevalidate.ValidationReport, error) {
21-
// Ran crane validate command and parse the report
22-
stdout, err := runner.Validate(ValidateOptions{
23-
InputDir: inputDir,
24-
ValidateDir: validateDir,
25-
APIResourcesFile: apiSurfaceFile,
26-
OutputFormat: outputFormat,
27-
})
28-
if err != nil {
29-
return cranevalidate.ValidationReport{}, err
30-
}
31-
log.Printf("Validate %s stdout: %s", outputFormat, stdout)
32-
33-
By("Verify " + outputFormat + " validation report exists")
34-
reportExt := "." + outputFormat
35-
reportPath := filepath.Join(validateDir, "report"+reportExt)
36-
Expect(reportPath).To(BeAnExistingFile(), "expected report%s at %s", reportExt, reportPath)
37-
38-
By("Parse and verify " + outputFormat + " validation report")
39-
reportData, err := os.ReadFile(reportPath)
40-
if err != nil {
41-
return cranevalidate.ValidationReport{}, err
42-
}
43-
44-
var report cranevalidate.ValidationReport
45-
if outputFormat == "yaml" {
46-
err = yaml.Unmarshal(reportData, &report)
47-
} else {
48-
err = json.Unmarshal(reportData, &report)
49-
}
50-
if err != nil {
51-
return cranevalidate.ValidationReport{}, err
52-
}
53-
54-
return report, nil
55-
}
56-
57-
func verifyValidateResults(report cranevalidate.ValidationReport, namespace string,
58-
validateDir string, apiSurfaceFile string, outputFormat string) {
59-
// Verify report data and creation of failures directory when required
60-
By(outputFormat + " report: Verify report shows offline mode")
61-
Expect(report.Mode).To(Equal("offline"), "expected validation mode to be 'offline' in %s report", outputFormat)
62-
log.Printf("%s validation mode: %s", outputFormat, report.Mode)
63-
64-
By(outputFormat + " report: Verify apiResourcesSource is set to the captured API surface file")
65-
Expect(report.APIResourcesSource).NotTo(BeEmpty(), "expected apiResourcesSource to be set in offline mode")
66-
Expect(report.APIResourcesSource).To(Equal(apiSurfaceFile), "expected apiResourcesSource to match API surface file path")
67-
log.Printf("API resources source: %s", report.APIResourcesSource)
68-
69-
By(outputFormat + " report: Verify resource count")
70-
Expect(report.TotalScanned).To(Equal(5), "expected exactly 5 resources scanned in %s report", outputFormat)
71-
Expect(report.Compatible).To(Equal(5), "expected all 5 resources to be compatible in %s report", outputFormat)
72-
Expect(report.Incompatible).To(Equal(0), "expected no incompatible resources in %s report", outputFormat)
73-
log.Printf("%s report - Total: %d, Compatible: %d, Incompatible: %d",
74-
outputFormat, report.TotalScanned, report.Compatible, report.Incompatible)
75-
76-
By(outputFormat + " report: Verify resource API version, namespace, status")
77-
expectedResources := map[string]string{
78-
"Deployment": "apps/v1",
79-
"Service": "v1",
80-
"ConfigMap": "v1",
81-
"Secret": "v1",
82-
"RoleBinding": "rbac.authorization.k8s.io/v1",
83-
}
84-
85-
foundResources := make(map[string]bool)
86-
for _, result := range report.Results {
87-
if expectedAPIVersion, expected := expectedResources[result.Kind]; expected {
88-
foundResources[result.Kind] = true
89-
Expect(result.APIVersion).To(Equal(expectedAPIVersion),
90-
"expected %s to have apiVersion %s in %s report", result.Kind, expectedAPIVersion, outputFormat)
91-
Expect(result.Status).To(Equal(cranevalidate.StatusOK),
92-
"expected %s to have status OK in %s report", result.Kind, outputFormat)
93-
Expect(result.Namespace).To(Equal(namespace),
94-
"expected %s to be in namespace %s in %s report", result.Kind, namespace, outputFormat)
95-
}
96-
log.Printf("✓ %s report: Found %s with apiVersion %s in namespace %s with status %s",
97-
outputFormat, result.Kind, result.APIVersion, result.Namespace, result.Status)
98-
}
99-
100-
By(outputFormat + " report: Verify all expected resources were found")
101-
var missingResources []string
102-
for kind := range expectedResources {
103-
if !foundResources[kind] {
104-
missingResources = append(missingResources, kind)
105-
}
106-
}
107-
Expect(missingResources).To(BeEmpty(),
108-
"expected to find all resources in %s validation results, missing: %v", outputFormat, missingResources)
109-
110-
By(outputFormat + " output: Verify no failures directory was created")
111-
failuresDir := filepath.Join(validateDir, "failures")
112-
_, err := os.Stat(failuresDir)
113-
Expect(os.IsNotExist(err)).To(BeTrue(),
114-
"expected no failures/ directory for all compatible resources")
115-
log.Printf("No failures directory created")
116-
}
117-
11818
var _ = Describe("Crane validate: all compatible standard resources in offline mode in JSON and YAML formats", func() {
11919
It("[MTA-831][MTA-848] Generate and validate crane validate report in JSON and YAML formats",
12020
Label("tier0", "validate"), func() {
@@ -218,10 +118,44 @@ var _ = Describe("Crane validate: all compatible standard resources in offline m
218118
for _, ft := range formats {
219119
By("Run crane validate in offline mode with output in " + ft.label + " format")
220120
validateDir := filepath.Join(paths.TempDir, ft.dirSuffix)
221-
report, err := runValidateAndParseReport(runner, paths.OutputDir, validateDir, apiSurfaceFile, ft.format)
222-
Expect(err).NotTo(HaveOccurred(), "validate with %s output should succeed for all compatible resources", ft.label)
223121

224-
verifyValidateResults(report, namespace, validateDir, apiSurfaceFile, ft.label)
122+
// Run crane validate command
123+
stdout, err := runner.Validate(ValidateOptions{
124+
InputDir: paths.OutputDir,
125+
ValidateDir: validateDir,
126+
APIResourcesFile: apiSurfaceFile,
127+
OutputFormat: ft.format,
128+
})
129+
Expect(err).NotTo(HaveOccurred(), "crane validate should succeed")
130+
log.Printf("Validate %s stdout: %s", ft.format, stdout)
131+
132+
By("Parse " + ft.label + " validation report")
133+
var report cranevalidate.ValidationReport
134+
err = utils.ParseValidationReport(validateDir, ft.format, &report)
135+
Expect(err).NotTo(HaveOccurred(), "should parse %s report", ft.label)
136+
137+
// Define expectations for this test
138+
expectations := utils.ValidationExpectations{
139+
ValidationReport: cranevalidate.ValidationReport{
140+
Mode: "offline",
141+
APIResourcesSource: apiSurfaceFile,
142+
TotalScanned: 5,
143+
Compatible: 5,
144+
Incompatible: 0,
145+
},
146+
ExpectedResources: map[string]string{
147+
"Deployment": "apps/v1",
148+
"Service": "v1",
149+
"ConfigMap": "v1",
150+
"Secret": "v1",
151+
"RoleBinding": "rbac.authorization.k8s.io/v1",
152+
},
153+
ExpectedStatus: cranevalidate.StatusOK,
154+
Namespace: namespace,
155+
ExpectFailuresDir: false,
156+
}
157+
158+
utils.VerifyValidateResults(report, validateDir, ft.label, expectations)
225159

226160
log.Printf("\n"+
227161
"========================================\n"+

‎e2e-tests/utils/utils.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,3 +902,39 @@ func CaptureAPISurfaceScriptPath() (string, error) {
902902

903903
return scriptPath, nil
904904
}
905+
906+
// ParseValidationReport reads and parses a crane validate report file.
907+
// It supports both JSON and YAML formats based on the outputFormat parameter.
908+
// The report parameter should be a pointer to the structure that will hold the parsed data.
909+
func ParseValidationReport(validateDir string, outputFormat string, report interface{}) error {
910+
if outputFormat != "json" && outputFormat != "yaml" {
911+
return fmt.Errorf("unsupported output format: %s (must be 'json' or 'yaml')", outputFormat)
912+
}
913+
914+
reportExt := "." + outputFormat
915+
reportPath := filepath.Join(validateDir, "report"+reportExt)
916+
917+
// Check if report file exists
918+
if _, err := os.Stat(reportPath); err != nil {
919+
return fmt.Errorf("report file not found at %s: %w", reportPath, err)
920+
}
921+
922+
// Read report file
923+
reportData, err := os.ReadFile(reportPath)
924+
if err != nil {
925+
return fmt.Errorf("failed to read report file: %w", err)
926+
}
927+
928+
// Parse based on format
929+
if outputFormat == "yaml" {
930+
if err := yaml.Unmarshal(reportData, report); err != nil {
931+
return fmt.Errorf("failed to parse YAML report: %w", err)
932+
}
933+
} else {
934+
if err := json.Unmarshal(reportData, report); err != nil {
935+
return fmt.Errorf("failed to parse JSON report: %w", err)
936+
}
937+
}
938+
939+
return nil
940+
}

‎e2e-tests/utils/utils_validate.go‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
// Package utils provides utilities for e2e tests.
2+
//
3+
// This file (utils_validate.go) contains Ginkgo/Gomega-specific test helpers
4+
// for validating crane validate command results. Unlike the framework-agnostic
5+
// utilities in utils.go (which return errors), these helpers use Ginkgo's Expect()
6+
// and By() for more convenient test assertions.
7+
//
8+
// These helpers are designed to be used across all test tiers (tier0, tier1, tier2+)
9+
// but are ONLY compatible with Ginkgo/Gomega test suites.
10+
package utils
11+
12+
import (
13+
"log"
14+
"os"
15+
"path/filepath"
16+
17+
"github.com/konveyor/crane/internal/validate"
18+
. "github.com/onsi/ginkgo/v2"
19+
. "github.com/onsi/gomega"
20+
)
21+
22+
// ValidationExpectations holds the expected values for validating a crane validate report.
23+
//
24+
// It embeds validate.ValidationReport to reuse standard fields (Mode, APIResourcesSource, TotalScanned, etc.)
25+
// and adds test-specific fields for verification.
26+
//
27+
// NOTE: This struct is designed to be used with VerifyValidateResults(), which is a Ginkgo-specific helper.
28+
type ValidationExpectations struct {
29+
validate.ValidationReport // Embedded: Mode, APIResourcesSource, TotalScanned, Compatible, Incompatible
30+
ExpectedResources map[string]string // Map of Kind -> APIVersion for expected resources (simpler than Results)
31+
ExpectedStatus validate.ValidationStatus // Expected status for resources (e.g., StatusOK)
32+
Namespace string // Expected namespace for resources
33+
ExpectFailuresDir bool // Whether to expect a failures/ directory
34+
}
35+
36+
// VerifyValidateResults verifies a crane validate report against expected values using Ginkgo assertions.
37+
//
38+
// IMPORTANT: This is a Ginkgo/Gomega-specific test helper. It uses Expect() and By() internally,
39+
// so it can ONLY be called from within Ginkgo test specs (It, BeforeEach, etc.).
40+
// For framework-agnostic validation, use ParseValidationReport() instead and write your own assertions.
41+
//
42+
// This helper can be used by any validate test in any mode (offline or live mode).
43+
//
44+
// Parameters:
45+
// - report: The parsed ValidationReport from crane validate
46+
// - validateDir: The directory where validate output was written
47+
// - outputFormat: The format label (e.g., "JSON", "YAML") for logging
48+
// - expectations: ValidationExpectations struct with expected values
49+
//
50+
// Example usage:
51+
//
52+
// expectations := utils.ValidationExpectations{
53+
// ValidationReport: validate.ValidationReport{
54+
// Mode: "offline",
55+
// APIResourcesSource: apiSurfaceFile,
56+
// TotalScanned: 5,
57+
// Compatible: 5,
58+
// Incompatible: 0,
59+
// },
60+
// ExpectedResources: map[string]string{
61+
// "Deployment": "apps/v1",
62+
// "Service": "v1",
63+
// },
64+
// ExpectedStatus: validate.StatusOK,
65+
// Namespace: "my-namespace",
66+
// ExpectFailuresDir: false,
67+
// }
68+
// utils.VerifyValidateResults(report, validateDir, "JSON", expectations)
69+
func VerifyValidateResults(report validate.ValidationReport, validateDir string,
70+
outputFormat string, expectations ValidationExpectations) {
71+
// Verify report mode
72+
By(outputFormat + " report: Verify report shows " + expectations.Mode + " mode")
73+
Expect(report.Mode).To(Equal(expectations.Mode), "expected validation mode to be '%s' in %s report", expectations.Mode, outputFormat)
74+
log.Printf("%s validation mode: %s", outputFormat, report.Mode)
75+
76+
// Verify apiResourcesSource (for offline mode)
77+
if expectations.Mode == "offline" && expectations.APIResourcesSource != "" {
78+
By(outputFormat + " report: Verify apiResourcesSource is set to the captured API surface file")
79+
Expect(report.APIResourcesSource).NotTo(BeEmpty(), "expected apiResourcesSource to be set in offline mode")
80+
Expect(report.APIResourcesSource).To(Equal(expectations.APIResourcesSource), "expected apiResourcesSource to match API surface file path")
81+
log.Printf("API resources source: %s", report.APIResourcesSource)
82+
}
83+
84+
// Verify resource counts
85+
By(outputFormat + " report: Verify resource count")
86+
Expect(report.TotalScanned).To(Equal(expectations.TotalScanned), "expected %d resources scanned in %s report", expectations.TotalScanned, outputFormat)
87+
Expect(report.Compatible).To(Equal(expectations.Compatible), "expected %d compatible resources in %s report", expectations.Compatible, outputFormat)
88+
Expect(report.Incompatible).To(Equal(expectations.Incompatible), "expected %d incompatible resources in %s report", expectations.Incompatible, outputFormat)
89+
log.Printf("%s report - Total: %d, Compatible: %d, Incompatible: %d",
90+
outputFormat, report.TotalScanned, report.Compatible, report.Incompatible)
91+
92+
// Verify expected resources if provided
93+
if len(expectations.ExpectedResources) > 0 {
94+
By(outputFormat + " report: Verify resource API version, namespace, status")
95+
foundResources := make(map[string]bool)
96+
for _, result := range report.Results {
97+
if expectedAPIVersion, expected := expectations.ExpectedResources[result.Kind]; expected {
98+
foundResources[result.Kind] = true
99+
Expect(result.APIVersion).To(Equal(expectedAPIVersion),
100+
"expected %s to have apiVersion %s in %s report", result.Kind, expectedAPIVersion, outputFormat)
101+
Expect(result.Status).To(Equal(expectations.ExpectedStatus),
102+
"expected %s to have status %s in %s report", result.Kind, expectations.ExpectedStatus, outputFormat)
103+
if expectations.Namespace != "" {
104+
Expect(result.Namespace).To(Equal(expectations.Namespace),
105+
"expected %s to be in namespace %s in %s report", result.Kind, expectations.Namespace, outputFormat)
106+
}
107+
}
108+
log.Printf("✓ %s report: Found %s with apiVersion %s in namespace %s with status %s",
109+
outputFormat, result.Kind, result.APIVersion, result.Namespace, result.Status)
110+
}
111+
112+
By(outputFormat + " report: Verify all expected resources were found")
113+
var missingResources []string
114+
for kind := range expectations.ExpectedResources {
115+
if !foundResources[kind] {
116+
missingResources = append(missingResources, kind)
117+
}
118+
}
119+
Expect(missingResources).To(BeEmpty(),
120+
"expected to find all resources in %s validation results, missing: %v", outputFormat, missingResources)
121+
}
122+
123+
// Verify failures directory
124+
By(outputFormat + " output: Verify failures directory expectation")
125+
failuresDir := filepath.Join(validateDir, "failures")
126+
_, err := os.Stat(failuresDir)
127+
if expectations.ExpectFailuresDir {
128+
Expect(err).NotTo(HaveOccurred(), "expected failures/ directory to exist")
129+
log.Printf("Failures directory created as expected")
130+
} else {
131+
Expect(os.IsNotExist(err)).To(BeTrue(), "expected no failures/ directory")
132+
log.Printf("No failures directory created")
133+
}
134+
}

0 commit comments

Comments
 (0)