Allows per test directory for generated files on failure (#10579)

Author: andy-fong

changelog/v1.19.0-beta4/allow-per-test-directory-for-generated-files.yaml

@@ -0,0 +1,6 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/7634
+    resolvesIssue: false
+    description: >-
+      Added optional option for PreFailHandler for dumping into per test directory

test/kubernetes/e2e/example/info_logging_test.go

@@ -46,6 +46,11 @@ func TestInstallationWithInfoLogLevel(t *testing.T) {
 		if !nsEnvPredefined {
 			os.Unsetenv(testutils.InstallNamespace)
 		}
+
+		// This clean up function is called only once after all the tests in all the registered suites are done.
+		// Moreover, PreFailHandler() will wipe out the output directory before dumping out the stats and
+		// logs of the cluster. If PreFailHandler() is called in the suite's AfterTest() function already,
+		// The following should be removed.
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
 		}

test/kubernetes/e2e/features/example/suite.go

@@ -44,6 +44,23 @@ func (s *testingSuite) BeforeTest(suiteName, testName string) {
 
 func (s *testingSuite) AfterTest(suiteName, testName string) {
 	// This is code that will be executed after each test is run
+
+	// PreFailHandler() logs the states of the clusters and dump out logs from various pods for debugging
+	// when the test fails. If the test create and remove resources that will spin up the pod and delete
+	// it afterward, PreFailHandler() should be called here (before the resources are deleted) so we can
+	// capture the log before the pod is destroyed.
+
+	// WARNING: In this example test, another place calling PreFailHandler() is in the main _test.go file
+	// (test/kubernetes/e2e/example/info_logging_test.go) where it sets up the go test cleanup function with t.Cleanup().
+	// That clean up function is only called once/ after all the tests in all registered suites finished.
+	// If PreFailHandler() is called here, the one in the cleanup function should be removed as it would wipe out the
+	// entire output directory (including these per-test logs) before dumping out the logs at the very end.
+	// If you only need to dump the logs once at the very end, the following should be removed.
+	if s.T().Failed() {
+		// Calling PreFailHandler() with optional TestName so that the logs would go into
+		// a per test directory
+		s.testInstallation.PreFailHandler(s.ctx, e2e.PreFailHandlerOption{TestName: testName})
+	}
 }
 
 func (s *testingSuite) TestExampleAssertion() {

test/kubernetes/e2e/test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -235,12 +236,19 @@ func (i *TestInstallation) UninstallGlooGateway(ctx context.Context, uninstallFn
 	i.Assertions.EventuallyUninstallationSucceeded(ctx)
 }
 
+type PreFailHandlerOption struct {
+	TestName string
+}
+
 // PreFailHandler is the function that is invoked if a test in the given TestInstallation fails
-func (i *TestInstallation) PreFailHandler(ctx context.Context) {
+func (i *TestInstallation) PreFailHandler(ctx context.Context, options ...PreFailHandlerOption) {
 	// The idea here is we want to accumulate ALL information about this TestInstallation into a single directory
 	// That way we can upload it in CI, or inspect it locally
 
 	failureDir := i.GeneratedFiles.FailureDir
+	if len(options) > 0 && len(options[0].TestName) > 0 {
+		failureDir = path.Join(failureDir, options[0].TestName)
+	}
 	err := os.Mkdir(failureDir, os.ModePerm)
 	// We don't want to fail on the output directory already existing. This could occur
 	// if multiple tests running in the same cluster from the same installation namespace