Skip to content

Commit d6ff78a

Browse files
sadasusadasu
committed
Update featuregate-test-analyzer output for Install features
Augment `verify-feature-promotion` output to indicate pass percentage for `install should succeed` tests for featuregates that include "Install" in their name. This update gives a better indication of whether Install features are failing at installation or later during execution of e2e conformance tests. This update does not change the criteria for reporting success but adds more information in the output for easier analysis of feature state.
1 parent 72066cc commit d6ff78a

2 files changed

Lines changed: 226 additions & 2 deletions

File tree

‎tools/codegen/cmd/featuregate-test-analyzer.go‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,27 @@ func checkIfTestingIsSufficient(featureGate string, testingResults map[JobVarian
415415
})
416416
}
417417
}
418+
419+
// For Install feature gates, report pass percentage for "install should succeed" test
420+
if strings.Contains(featureGate, "Install") {
421+
installTest := testResultByName(testedVariant.TestResults, "install should succeed")
422+
if installTest == nil {
423+
// Missing "install should succeed" test for an Install feature gate is a warning
424+
results = append(results, ValidationResult{
425+
Error: fmt.Errorf("warning: \"install should succeed\" test not found for Install feature gate %q on %v",
426+
featureGate, jobVariant),
427+
IsWarning: true,
428+
})
429+
} else if installTest.TotalRuns > 0 {
430+
passPercent := float32(installTest.SuccessfulRuns) / float32(installTest.TotalRuns)
431+
displayActual := int(passPercent * 100)
432+
results = append(results, ValidationResult{
433+
Error: fmt.Errorf("info: \"install should succeed\" test passed %d%% (%d/%d runs) for %q on %v",
434+
displayActual, installTest.SuccessfulRuns, installTest.TotalRuns, featureGate, jobVariant),
435+
IsWarning: true, // Info messages are non-blocking
436+
})
437+
}
438+
}
418439
}
419440

420441
return results
@@ -1139,7 +1160,7 @@ func getJobsForFeatureGateFromSippy(client *http.Client, release, featureGate st
11391160
defer resp.Body.Close()
11401161

11411162
if resp.StatusCode != http.StatusOK {
1142-
return nil, fmt.Errorf("expected a 200 OK status code but got %s", resp.StatusCode)
1163+
return nil, fmt.Errorf("expected a 200 OK status code but got %d", resp.StatusCode)
11431164
}
11441165

11451166
body, err := io.ReadAll(resp.Body)
@@ -1165,7 +1186,7 @@ func getJobRunsFromSippy(client *http.Client, release, jobName string) ([]sippy.
11651186
defer resp.Body.Close()
11661187

11671188
if resp.StatusCode != http.StatusOK {
1168-
return nil, fmt.Errorf("expected a 200 OK status code but got %s", resp.StatusCode)
1189+
return nil, fmt.Errorf("expected a 200 OK status code but got %d", resp.StatusCode)
11691190
}
11701191

11711192
body, err := io.ReadAll(resp.Body)

‎tools/codegen/cmd/featuregate-test-analyzer_test.go‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,209 @@ func Test_checkIfTestingIsSufficient_OptionalVariants(t *testing.T) {
481481
}
482482
}
483483

484+
func Test_checkIfTestingIsSufficient_InstallFeatureGates(t *testing.T) {
485+
tests := []struct {
486+
name string
487+
featureGate string
488+
testingResults map[JobVariant]*TestingResults
489+
wantBlockingErrors int
490+
wantWarnings int
491+
wantInfoMessages int
492+
}{
493+
{
494+
name: "Install feature gate with install should succeed test - info message reported",
495+
featureGate: "FakeInstallFeature",
496+
testingResults: map[JobVariant]*TestingResults{
497+
{
498+
Cloud: "metal",
499+
Architecture: "amd64",
500+
Topology: "ha",
501+
}: {
502+
TestResults: []TestResults{
503+
{TestName: "install should succeed", TotalRuns: 20, SuccessfulRuns: 19},
504+
{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
505+
{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
506+
{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
507+
{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
508+
},
509+
},
510+
},
511+
wantBlockingErrors: 0,
512+
wantWarnings: 1, // Info message about install test pass percentage
513+
wantInfoMessages: 1,
514+
},
515+
{
516+
name: "Install feature gate without install should succeed test - warning reported",
517+
featureGate: "MockInstallGate",
518+
testingResults: map[JobVariant]*TestingResults{
519+
{
520+
Cloud: "metal",
521+
Architecture: "amd64",
522+
Topology: "ha",
523+
}: {
524+
TestResults: []TestResults{
525+
{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
526+
{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
527+
{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
528+
{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
529+
{TestName: "test5", TotalRuns: 15, SuccessfulRuns: 15},
530+
},
531+
},
532+
},
533+
wantBlockingErrors: 0,
534+
wantWarnings: 1, // Warning about missing "install should succeed" test
535+
wantInfoMessages: 0,
536+
},
537+
{
538+
name: "Non-Install feature gate with install should succeed test - no info message",
539+
featureGate: "SomeOtherFeature",
540+
testingResults: map[JobVariant]*TestingResults{
541+
{
542+
Cloud: "aws",
543+
Architecture: "amd64",
544+
Topology: "ha",
545+
}: {
546+
TestResults: []TestResults{
547+
{TestName: "install should succeed", TotalRuns: 20, SuccessfulRuns: 19},
548+
{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
549+
{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
550+
{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
551+
{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
552+
},
553+
},
554+
},
555+
wantBlockingErrors: 0,
556+
wantWarnings: 0,
557+
wantInfoMessages: 0,
558+
},
559+
{
560+
name: "Install feature gate with perfect pass rate - 100% reported",
561+
featureGate: "FakeInstallFeature",
562+
testingResults: map[JobVariant]*TestingResults{
563+
{
564+
Cloud: "aws",
565+
Architecture: "amd64",
566+
Topology: "ha",
567+
}: {
568+
TestResults: []TestResults{
569+
{TestName: "install should succeed", TotalRuns: 20, SuccessfulRuns: 20},
570+
{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
571+
{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
572+
{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
573+
{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
574+
},
575+
},
576+
},
577+
wantBlockingErrors: 0,
578+
wantWarnings: 1, // Info message
579+
wantInfoMessages: 1,
580+
},
581+
{
582+
name: "Install feature gate with multiple variants - info for each",
583+
featureGate: "FakeInstallFeature",
584+
testingResults: map[JobVariant]*TestingResults{
585+
{
586+
Cloud: "metal",
587+
Architecture: "amd64",
588+
Topology: "ha",
589+
}: {
590+
TestResults: []TestResults{
591+
{TestName: "install should succeed", TotalRuns: 20, SuccessfulRuns: 19},
592+
{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
593+
{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
594+
{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
595+
{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
596+
},
597+
},
598+
{
599+
Cloud: "metal",
600+
Architecture: "amd64",
601+
Topology: "single",
602+
}: {
603+
TestResults: []TestResults{
604+
{TestName: "install should succeed", TotalRuns: 18, SuccessfulRuns: 18},
605+
{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
606+
{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
607+
{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
608+
{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
609+
},
610+
},
611+
},
612+
wantBlockingErrors: 0,
613+
wantWarnings: 2, // Info message for each variant
614+
wantInfoMessages: 2,
615+
},
616+
{
617+
name: "Install feature gate with zero runs for install test - blocking error plus no info message",
618+
featureGate: "MockInstallGate",
619+
testingResults: map[JobVariant]*TestingResults{
620+
{
621+
Cloud: "aws",
622+
Architecture: "amd64",
623+
Topology: "ha",
624+
}: {
625+
TestResults: []TestResults{
626+
{TestName: "install should succeed", TotalRuns: 0, SuccessfulRuns: 0},
627+
{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
628+
{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
629+
{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
630+
{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
631+
},
632+
},
633+
},
634+
wantBlockingErrors: 1, // Blocking error for insufficient runs
635+
wantWarnings: 0,
636+
wantInfoMessages: 0,
637+
},
638+
}
639+
640+
for _, tt := range tests {
641+
t.Run(tt.name, func(t *testing.T) {
642+
results := checkIfTestingIsSufficient(tt.featureGate, tt.testingResults)
643+
644+
blockingErrors := 0
645+
warnings := 0
646+
infoMessages := 0
647+
for _, result := range results {
648+
if result.IsWarning {
649+
warnings++
650+
// Check if this is an info message about install test
651+
if strings.Contains(result.Error.Error(), "info:") && strings.Contains(result.Error.Error(), "install should succeed") {
652+
infoMessages++
653+
}
654+
} else {
655+
blockingErrors++
656+
}
657+
}
658+
659+
if blockingErrors != tt.wantBlockingErrors {
660+
t.Errorf("got %d blocking errors, want %d", blockingErrors, tt.wantBlockingErrors)
661+
for _, result := range results {
662+
if !result.IsWarning {
663+
t.Logf(" Blocking error: %v", result.Error)
664+
}
665+
}
666+
}
667+
if warnings != tt.wantWarnings {
668+
t.Errorf("got %d warnings, want %d", warnings, tt.wantWarnings)
669+
for _, result := range results {
670+
if result.IsWarning {
671+
t.Logf(" Warning: %v", result.Error)
672+
}
673+
}
674+
}
675+
if infoMessages != tt.wantInfoMessages {
676+
t.Errorf("got %d info messages, want %d", infoMessages, tt.wantInfoMessages)
677+
for _, result := range results {
678+
if result.IsWarning && strings.Contains(result.Error.Error(), "info:") {
679+
t.Logf(" Info: %v", result.Error)
680+
}
681+
}
682+
}
683+
})
684+
}
685+
}
686+
484687
func Test_defaultQueriesIncludeCandidateTier(t *testing.T) {
485688
// When JobTiers is empty, QueriesFor should generate queries for all tiers
486689
// including candidate. This test is added to prevent regressions for candidate-tier

0 commit comments

Comments
 (0)