Mitigate intermittent SSL runner timeouts on FreeBSD CI (#3171)

### Description of changes:

The FreeBSD 15.0 (x86-64) CI job fails intermittently on CBC record
splitting tests (`CBCRecordSplitting-AES128`,
`CBCRecordSplittingPartialWrite-AES128`) with runner-side TCP timeouts
like `read tcp [::1]:...->[::1]:...: i/o timeout`. The QEMU-emulated
FreeBSD VM is occasionally too slow to respond within the runner's
default 30-second idle timeout.

The runner already has retry logic for shim-side idle timeouts and
transient signals, but runner-side TCP timeouts (`net.Error` with
`Timeout() == true`) fall through without a retry — the test just fails.

This PR does two things:
1. Increases the idle timeout to 120s and enables retry-on-timeout for
the FreeBSD CI job.
2. Adds a new `-retry-on-timeout` flag to the SSL test runner that
retries tests on runner-side TCP timeouts. A real bug will fail again on
the retry.

### Call-outs:

The `-retry-on-timeout` flag is off by default. It's gated behind
`AWS_LC_SSL_RUNNER_RETRY_ON_TIMEOUT` at the CMake level, currently only
enabled for FreeBSD. If similar flakiness shows up on NetBSD or OpenBSD,
we can add the same env vars to those jobs.

This PR also fixes a pre-existing variable shadowing bug in the retry
block: `shim, err := newShimProcess(...)` was creating a new `shim`
scoped to the `if` block, so after a successful retry the code would
still read `stdout`/`stderr` from the original failed shim. This caused
retried tests to fail with `unexpected error output` even when the retry
passed. Changed to `shim, err = ...` to reassign the outer variable.

### Testing:

This addresses a CI flakiness issue — the fix will be validated by
observing the FreeBSD job over subsequent runs. The retry logic is
structurally identical to the existing `shim.idled` retry path.



By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.

Author: justsmth

.github/workflows/actions-ci.yml

@@ -711,9 +711,11 @@ jobs:
         uses: cross-platform-actions/action@v0.32.0
         env:
           AWS_LC_GO_TEST_TIMEOUT: 90m
+          AWS_LC_SSL_RUNNER_IDLE_TIMEOUT: 120s
+          AWS_LC_SSL_RUNNER_RETRY_ON_TIMEOUT: 1
           GOFLAGS: "-buildvcs=false"
         with:
-          environment_variables: "AWS_LC_GO_TEST_TIMEOUT GOFLAGS"
+          environment_variables: "AWS_LC_GO_TEST_TIMEOUT AWS_LC_SSL_RUNNER_IDLE_TIMEOUT AWS_LC_SSL_RUNNER_RETRY_ON_TIMEOUT GOFLAGS"
           operating_system: freebsd
           architecture: ${{ matrix.arch }}
           version: ${{ matrix.version }}

CMakeLists.txt

@@ -1350,6 +1350,13 @@ if(DEFINED ENV{AWS_LC_SSL_RUNNER_IDLE_TIMEOUT})
   set(RUNNER_ARGS ${RUNNER_ARGS} "-idle-timeout" "$ENV{AWS_LC_SSL_RUNNER_IDLE_TIMEOUT}")
 endif()
 
+# Allow enabling retry on runner-side TCP timeouts.
+# Useful for resource-constrained environments (e.g. emulated FreeBSD VMs) where
+# transient timeouts do not indicate real bugs.
+if(DEFINED ENV{AWS_LC_SSL_RUNNER_RETRY_ON_TIMEOUT})
+  set(RUNNER_ARGS ${RUNNER_ARGS} "-retry-on-timeout")
+endif()
+
 if (NOT ${CMAKE_VERSION} VERSION_LESS "3.2")
   # USES_TERMINAL is only available in CMake 3.2 or later.
   set(MAYBE_USES_TERMINAL USES_TERMINAL)

ssl/test/runner/runner.go

@@ -69,6 +69,7 @@ var (
 	includeDisabled    = flag.Bool("include-disabled", false, "If true, also runs disabled tests.")
 	repeatUntilFailure = flag.Bool("repeat-until-failure", false, "If true, the first selected test will be run repeatedly until failure.")
 	// Added by aws-lc
+	retryOnTimeout     = flag.Bool("retry-on-timeout", false, "If true, retry tests that fail due to a runner-side TCP timeout. Useful for resource-constrained environments (e.g. emulated VMs).")
 	sslTransferConfig  = flag.String("ssl-transfer-test-file", "", "A path to file which includes the test names that can be converted for SSL transfer.")
 	sslFuzzSeedDir     = flag.String("ssl-fuzz-seed-dir", "", "The directory in which to write the output of |SSL_to_bytes|.")
 	testCaseStartIndex = flag.Int("test-case-start-index", -1, "If non-negative, test case is filtered in if the index in |testCases| >= test-case-start-index.")
@@ -1671,17 +1672,25 @@ func runTest(dispatcher *shimDispatcher, statusChan chan statusMsg, test *testCa
 	var retryReason string
 	if shim.idled {
 		retryReason = "idle timeout"
-	} else if exitErr, ok := childErr.(*exec.ExitError); ok {
-		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() {
-			switch status.Signal() {
-			case syscall.SIGABRT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGPIPE:
-				retryReason = fmt.Sprintf("signal: %s", status.Signal())
+	} else if *retryOnTimeout {
+		if netErr, ok := localErr.(net.Error); ok && netErr.Timeout() {
+			retryReason = "local timeout"
+		}
+	}
+	if retryReason == "" {
+		if exitErr, ok := childErr.(*exec.ExitError); ok {
+			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() {
+				switch status.Signal() {
+				case syscall.SIGABRT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGPIPE:
+					retryReason = fmt.Sprintf("signal: %s", status.Signal())
+				}
 			}
 		}
 	}
 	if retryReason != "" {
 		fmt.Fprintf(os.Stderr, "Retrying %s (%s)\n", test.name, retryReason)
-		shim, err := newShimProcess(dispatcher, shimPath, flags, env)
+		var err error
+		shim, err = newShimProcess(dispatcher, shimPath, flags, env)
 		if err != nil {
 			return err
 		}