feat: add test suite and test case parallelization

Author: florian.cazals

cmd/venom/run/cmd.go

@@ -25,24 +25,26 @@ var (
 	path []string
 	v    *venom.Venom
 
-	variables     []string
-	secrets       []string
-	format        string = "xml" // Set the default value for formatFlag
-	varFiles      []string
-	outputDir     string
-	libDir        string
-	htmlReport    bool
-	stopOnFailure bool
-	verbose       int = 0 // Set the default value for verboseFlag
-
-	variablesFlag     *[]string
-	formatFlag        *string
-	varFilesFlag      *[]string
-	outputDirFlag     *string
-	libDirFlag        *string
-	stopOnFailureFlag *bool
-	htmlReportFlag    *bool
-	verboseFlag       *int
+	variables      []string
+	secrets        []string
+	format         string = "xml" // Set the default value for formatFlag
+	varFiles       []string
+	outputDir      string
+	libDir         string
+	htmlReport     bool
+	stopOnFailure  bool
+	verbose        int = 0 // Set the default value for verboseFlag
+	parallelSuites int = 0 // Number of testsuites to run in parallel (0 = sequential)
+
+	variablesFlag      *[]string
+	formatFlag         *string
+	varFilesFlag       *[]string
+	outputDirFlag      *string
+	libDirFlag         *string
+	stopOnFailureFlag  *bool
+	htmlReportFlag     *bool
+	verboseFlag        *int
+	parallelSuitesFlag *int
 )
 
 func init() {
@@ -54,6 +56,7 @@ func init() {
 	variablesFlag = Cmd.Flags().StringArray("var", nil, "--var cds='cds -f config.json' --var cds2='cds -f config.json'")
 	outputDirFlag = Cmd.PersistentFlags().String("output-dir", "", "Output Directory: create tests results file inside this directory")
 	libDirFlag = Cmd.PersistentFlags().String("lib-dir", "", "Lib Directory: can contain user executors. example:/etc/venom/lib:$HOME/venom.d/lib")
+	parallelSuitesFlag = Cmd.Flags().Int("parallel-suites", 0, "Number of testsuites to run in parallel (0 or 1 = sequential)")
 }
 
 func initArgs(cmd *cobra.Command) {
@@ -115,6 +118,10 @@ func initFromCommandArguments(f *pflag.Flag) {
 				variables = mergeVariables(varFlag, variables)
 			}
 		}
+	case "parallel-suites":
+		if parallelSuitesFlag != nil {
+			parallelSuites = *parallelSuitesFlag
+		}
 	}
 }
 
@@ -161,6 +168,7 @@ type ConfigFileData struct {
 	Secrets        *[]string `json:"secrets,omitempty" yaml:"secrets,omitempty"`
 	VariablesFiles *[]string `json:"variables_files,omitempty" yaml:"variables_files,omitempty"`
 	Verbosity      *int      `json:"verbosity,omitempty" yaml:"verbosity,omitempty"`
+	ParallelSuites *int      `json:"parallel_suites,omitempty" yaml:"parallel_suites,omitempty"`
 }
 
 // Configuration file overrides the environment variables.
@@ -209,6 +217,9 @@ func initFromReaderConfigFile(reader io.Reader) error {
 	if configFileData.Verbosity != nil {
 		verbose = *configFileData.Verbosity
 	}
+	if configFileData.ParallelSuites != nil {
+		parallelSuites = *configFileData.ParallelSuites
+	}
 
 	return nil
 }
@@ -279,6 +290,13 @@ func initFromEnv(environ []string) ([]string, error) {
 		v2 := int(v)
 		verbose = v2
 	}
+	if os.Getenv("VENOM_PARALLEL_SUITES") != "" {
+		v, err := strconv.ParseInt(os.Getenv("VENOM_PARALLEL_SUITES"), 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("invalid value for VENOM_PARALLEL_SUITES, must be a positive integer")
+		}
+		parallelSuites = int(v)
+	}
 
 	for _, env := range environ {
 		if strings.HasPrefix(env, "VENOM_VAR_") {
@@ -299,6 +317,7 @@ func displayArg(ctx context.Context) {
 	venom.Debug(ctx, "option htmlReport=%v", htmlReport)
 	venom.Debug(ctx, "option varFiles=%v", strings.Join(varFiles, " "))
 	venom.Debug(ctx, "option verbose=%v", verbose)
+	venom.Debug(ctx, "option parallelSuites=%v", parallelSuites)
 }
 
 // Cmd run
@@ -338,6 +357,7 @@ var Cmd = &cobra.Command{
 		v.StopOnFailure = stopOnFailure
 		v.HtmlReport = htmlReport
 		v.Verbose = verbose
+		v.ParallelSuites = parallelSuites
 
 		if err := v.InitLogger(); err != nil {
 			fmt.Fprintf(os.Stderr, "%v\n", err)

process.go

@@ -1,11 +1,13 @@
 package venom
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	nested "github.com/antonfisher/nested-logrus-formatter"
@@ -183,17 +185,25 @@ func (v *Venom) Process(ctx context.Context, path []string) error {
 	v.Tests.Status = StatusRun
 	v.Tests.Start = time.Now()
 	Debug(ctx, "nb testsuites: %d", len(v.Tests.TestSuites))
-	for i := range v.Tests.TestSuites {
 
-		v.Tests.TestSuites[i].Start = time.Now()
-		// ##### RUN Test Suite Here
-		if err := v.runTestSuite(ctx, &v.Tests.TestSuites[i]); err != nil {
+	parallelSuites := v.ParallelSuites
+	if parallelSuites <= 1 {
+		// Sequential execution (default)
+		for i := range v.Tests.TestSuites {
+			v.Tests.TestSuites[i].Start = time.Now()
+			if err := v.runTestSuite(ctx, &v.Tests.TestSuites[i]); err != nil {
+				return err
+			}
+			v.Tests.TestSuites[i].End = time.Now()
+			v.Tests.TestSuites[i].Duration = v.Tests.TestSuites[i].End.Sub(v.Tests.TestSuites[i].Start).Seconds()
+		}
+	} else {
+		// Parallel execution of testsuites
+		if err := v.processTestSuitesParallel(ctx, parallelSuites); err != nil {
 			return err
 		}
-
-		v.Tests.TestSuites[i].End = time.Now()
-		v.Tests.TestSuites[i].Duration = v.Tests.TestSuites[i].End.Sub(v.Tests.TestSuites[i].Start).Seconds()
 	}
+
 	v.Tests.End = time.Now()
 	v.Tests.Duration = v.Tests.End.Sub(v.Tests.Start).Seconds()
 
@@ -219,3 +229,66 @@ func (v *Venom) Process(ctx context.Context, path []string) error {
 
 	return nil
 }
+
+// processTestSuitesParallel runs testsuites concurrently with a bounded worker pool.
+func (v *Venom) processTestSuitesParallel(ctx context.Context, maxWorkers int) error {
+	type suiteResult struct {
+		idx    int
+		err    error
+		output string
+	}
+
+	jobs := make(chan int, len(v.Tests.TestSuites))
+	results := make(chan suiteResult, len(v.Tests.TestSuites))
+
+	workerCount := maxWorkers
+	if workerCount > len(v.Tests.TestSuites) {
+		workerCount = len(v.Tests.TestSuites)
+	}
+
+	var wg sync.WaitGroup
+	for w := 0; w < workerCount; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for idx := range jobs {
+				ts := &v.Tests.TestSuites[idx]
+
+				// Buffer output to avoid interleaving between suites
+				var buf bytes.Buffer
+				vCopy := *v
+				vCopy.PrintFunc = func(format string, a ...interface{}) (n int, err error) {
+					return fmt.Fprintf(&buf, format, a...)
+				}
+
+				ts.Start = time.Now()
+				err := vCopy.runTestSuite(ctx, ts)
+				ts.End = time.Now()
+				ts.Duration = ts.End.Sub(ts.Start).Seconds()
+
+				results <- suiteResult{idx: idx, err: err, output: buf.String()}
+			}
+		}()
+	}
+
+	// Feed jobs
+	for i := range v.Tests.TestSuites {
+		jobs <- i
+	}
+	close(jobs)
+
+	// Collect results in completion order, print output serially
+	var firstErr error
+	for range v.Tests.TestSuites {
+		res := <-results
+		if res.output != "" {
+			v.Print("%s", res.output)
+		}
+		if res.err != nil && firstErr == nil {
+			firstErr = res.err
+		}
+	}
+
+	wg.Wait()
+	return firstErr
+}

process_files.go

@@ -124,6 +124,7 @@ func (v *Venom) readFiles(ctx context.Context, filesPath []string) (err error) {
 			TestCases:   make([]TestCase, len(testSuiteInput.TestCases)),
 			Vars:        testSuiteInput.Vars,
 			Secrets:     testSuiteInput.Secrets,
+			Parallel:    testSuiteInput.Parallel,
 		}
 		for i := range testSuiteInput.TestCases {
 			ts.TestCases[i] = TestCase{