Merge pull request ghostunnel#662 from ghostunnel/cs/better-tests

Parallelize integration test runner and handle timeouts better

Author: csstaub

CONTRIBUTING.md

@@ -52,6 +52,13 @@ run checks on a live instance. If you are adding new features or changing
 existing behavior, please add/update the integration tests in the `tests/`
 directory accordingly. The tests use the `tests/common.py` helper module.
 
+Integration tests run in parallel by default (up to `NumCPU`, capped at 16 by default).
+Set `GHOSTUNNEL_TEST_PARALLEL` to control the number of concurrent tests (may exceed the default cap):
+
+```bash
+GHOSTUNNEL_TEST_PARALLEL=4 mage test:integration
+```
+
 Test certificates are generated in `test-keys/` via `mage test:keys`.
 
 ## Architecture Overview

magefile.go

@@ -12,6 +12,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -375,7 +376,8 @@ func (Test) Unit(ctx context.Context) error {
 	return sh.Run("go", "test", "-v", "-covermode=count", "-coverprofile=coverage/unit-test.profile", "./...")
 }
 
-// Integration runs the integration tests.
+// Integration runs the integration tests in parallel.
+// Set GHOSTUNNEL_TEST_PARALLEL to control concurrency (default: NumCPU, max 16).
 func (Test) Integration(ctx context.Context) error {
 	mg.CtxDeps(ctx, Test.build)
 
@@ -395,47 +397,93 @@ func (Test) Integration(ctx context.Context) error {
 		return fmt.Errorf("failed to find test files: %w", err)
 	}
 
-	// Run each integration test directly
-	printf("Running integration tests...\n")
-	for _, testFile := range testFiles {
-		if err := ctx.Err(); err != nil {
-			return fmt.Errorf("context cancelled: %w", err)
+	// Determine parallelism
+	parallel := runtime.NumCPU()
+	if parallel > 16 {
+		parallel = 16
+	}
+	if envVal := os.Getenv("GHOSTUNNEL_TEST_PARALLEL"); envVal != "" {
+		if n, err := strconv.Atoi(envVal); err == nil && n > 0 {
+			parallel = n
 		}
+	}
 
-		testName := strings.TrimSuffix(filepath.Base(testFile), ".py")
-		printf("=== RUN   %s\n", testName)
-
-		// Run the Python test file directly from tests directory
-		start := time.Now()
-		testFileName := filepath.Base(testFile)
-		cmd := exec.CommandContext(ctx, "python3", testFileName)
-		cmd.Dir = "tests"
+	printf("Running %d integration tests with parallelism=%d...\n", len(testFiles), parallel)
 
-		// Capture stdout and stderr
-		var stdout, stderr bytes.Buffer
-		cmd.Stdout = &stdout
-		cmd.Stderr = &stderr
+	type testResult struct {
+		name     string
+		stdout   []byte
+		stderr   []byte
+		err      error
+		duration time.Duration
+	}
 
-		err := cmd.Run()
-		duration := time.Since(start)
-		elapsed := duration.Seconds()
+	// Channel-based semaphore for limiting concurrency
+	sem := make(chan struct{}, parallel)
+	results := make(chan testResult, len(testFiles))
 
-		if err == nil {
-			printf("=== PASS: %s (%.2fs)\n", testName, elapsed)
-			continue
+	// Launch all tests as goroutines
+	for _, testFile := range testFiles {
+		go func() {
+			// Check for context cancellation before acquiring semaphore
+			select {
+			case <-ctx.Done():
+				results <- testResult{
+					name: strings.TrimSuffix(filepath.Base(testFile), ".py"),
+					err:  ctx.Err(),
+				}
+				return
+			case sem <- struct{}{}: // acquire
+			}
+			defer func() { <-sem }() // release
+
+			testName := strings.TrimSuffix(filepath.Base(testFile), ".py")
+
+			start := time.Now()
+			testFileName := filepath.Base(testFile)
+			cmd := exec.CommandContext(ctx, "python3", testFileName)
+			cmd.Dir = "tests"
+
+			var stdout, stderr bytes.Buffer
+			cmd.Stdout = &stdout
+			cmd.Stderr = &stderr
+
+			err := cmd.Run()
+			duration := time.Since(start)
+
+			results <- testResult{
+				name:     testName,
+				stdout:   stdout.Bytes(),
+				stderr:   stderr.Bytes(),
+				err:      err,
+				duration: duration,
+			}
+		}()
+	}
+
+	// Collect results
+	var failed []testResult
+	for i := 0; i < len(testFiles); i++ {
+		r := <-results
+		if r.err == nil {
+			printf("=== PASS: %s (%.2fs)\n", r.name, r.duration.Seconds())
+		} else {
+			fmt.Printf("=== FAIL: %s (%.2fs)\n", r.name, r.duration.Seconds())
+			failed = append(failed, r)
 		}
+	}
 
-		// Test failed - output captured stdout/stderr and failure message
-		os.Stdout.Write(stdout.Bytes())
-		os.Stderr.Write(stderr.Bytes())
-		printf("=== FAIL: %s (%.2fs)\n", testName, elapsed)
-
-		// Get exit code if available
-		if exitError, ok := err.(*exec.ExitError); ok {
-			return fmt.Errorf("integration test %s failed with exit code %d", testName, exitError.ExitCode())
+	// Report failures
+	if len(failed) > 0 {
+		fmt.Printf("\n--- FAILURES ---\n")
+		for _, r := range failed {
+			fmt.Printf("\n=== FAIL: %s (%.2fs)\n", r.name, r.duration.Seconds())
+			fmt.Printf("--- stdout ---\n")
+			os.Stdout.Write(r.stdout)
+			fmt.Printf("--- stderr ---\n")
+			os.Stdout.Write(r.stderr)
 		}
-
-		return fmt.Errorf("integration test %s failed: %w", testName, err)
+		return fmt.Errorf("%d integration test(s) failed", len(failed))
 	}
 
 	return nil
@@ -498,8 +546,10 @@ func (Test) Single(ctx context.Context, name string) error {
 
 	// On failure, show captured output if not already streaming
 	if !mg.Verbose() {
+		fmt.Printf("--- stdout ---\n")
 		os.Stdout.Write(stdout.Bytes())
-		os.Stderr.Write(stderr.Bytes())
+		fmt.Printf("--- stderr ---\n")
+		os.Stdout.Write(stderr.Bytes())
 	}
 	fmt.Printf("=== FAIL: %s (%.2fs)\n", name, elapsed)
 	if exitError, ok := err.(*exec.ExitError); ok {
@@ -680,12 +730,21 @@ func (Test) Docker(ctx context.Context) error {
 		return fmt.Errorf("failed to get current directory: %w", err)
 	}
 
-	args = []string{"run", "-v", fmt.Sprintf("%s:/go/src/github.com/ghostunnel/ghostunnel", pwd), "ghostunnel/ghostunnel-test", "--"}
+	containerName := fmt.Sprintf("ghostunnel-test-%d", os.Getpid())
+	args = []string{"run", "--rm", "--name", containerName, "-v", fmt.Sprintf("%s:/go/src/github.com/ghostunnel/ghostunnel", pwd), "ghostunnel/ghostunnel-test", "--"}
 	if mg.Verbose() {
 		args = append(args, "-v")
 	}
 	args = append(args, "test:softhsmimport", "test:all")
-	return sh.Run("docker", args...)
+
+	defer func() {
+		exec.Command("docker", "rm", "-f", containerName).Run()
+	}()
+
+	cmd := exec.CommandContext(ctx, "docker", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
 }
 
 // Build builds and tags all Docker containers.

main.go

@@ -773,10 +773,9 @@ func (env *Environment) serveStatus() error {
 			}
 
 			logger.Printf("shutdown was requested via status endpoint")
+			w.WriteHeader(http.StatusOK)
 
 			env.shutdownChannel <- true
-
-			w.WriteHeader(http.StatusOK)
 		})
 	}