Copybara import of the project:

--
4eb3f49 by Jonathan Wright <jonathan@almalinux.org>:

Build test_suites concurrently in local_build.sh with a capped pool

Default cap is (nproc-1)/3 (floor 1), matching the ~3 hot worker
processes each `go test -c` spawns (compile, vet, link). Pass -j N to
override; -j 0 restores the auto formula.

A FIFO semaphore throttles to $jobs concurrent suites; each suite still
builds its 4 GOOS/GOARCH variants serially inside the slot. On a
32-core host this cuts a cold all-suites build from ~5m29s to ~2m30s
(-54%) and self-throttles cleanly on smaller hardware (4 cores → K=1,
effectively serial).

FUTURE_COPYBARA_INTEGRATE_REVIEW=#510 from jonathanspw:local_build_parallel 4eb3f49
PiperOrigin-RevId: 926948257

Author: jonathanspw

local_build.sh

@@ -13,23 +13,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# usage: local_build.sh -o $outspath -i $imagetestroot -s $suites_to_build
+# usage: local_build.sh -o $outspath -i $imagetestroot -s $suites_to_build -j $jobs
 # the output path of the test files
 outpath=.
 # the suites to build, space separated. all suites are built by default
 suites=*
 # the path to the imagetest folder, default value assumes this script is run from the imagetest folder. If set, the commands cd $imagetestroot/cmd and cd $imagetestroot/test_suites should succeed.
 imagetestroot=.
+# max suites to build concurrently. 0 means auto: max(1, (nproc-1)/3),
+# on the theory that each `go test -c` invocation spawns ~3 hot worker
+# processes (compile, vet, link), so K*3 <= cores-1. Pass -j N to override.
+jobs=0
 
 
-while getopts "o:s:i:" arg; do
-  case $arg in 
+while getopts "o:s:i:j:" arg; do
+  case $arg in
     o) outpath=$OPTARG;;
     s) suites=$OPTARG;;
     i) imagetestroot=$OPTARG;;
+    j) jobs=$OPTARG;;
     *) echo "unknown arg"
   esac
-done 
+done
+
+if [ "$jobs" -eq 0 ]; then
+  jobs=$(( ($(nproc) - 1) / 3 ))
+  [ $jobs -lt 1 ] && jobs=1
+fi
 
 
 export CGO_ENABLED=0
@@ -46,19 +56,58 @@ GOOS=windows GOARCH=386 go build -o $outpath/wrapp32.exe ./cmd/wrapper/main.go |
 go build -o $outpath/manager ./cmd/manager/main.go || exit 1
 
 
-cd test_suites
-for suite in $suites; do
-  [[ -d $suite ]] || continue
-  cd $suite
+# Build one suite (all four arch variants). Run in its own subshell so cd
+# stays scoped and so multiple suites can run concurrently.
+build_suite() {
+  local suite=$1
+  local outpath=$2
+  local root=$3
+  cd "$root/test_suites/$suite" || return 1
   echo "building suite $suite"
-  go test -c -tags cit || exit 1
-  ./"${suite}.test" -test.list '.*' > $outpath/"${suite}_tests.txt" || exit 1
-  mv "${suite}.test" $outpath/"${suite}.amd64.test" || exit 1
-  GOARCH=arm64 go test -c -tags cit || exit 1
-  mv "${suite}.test" "$outpath/${suite}.arm64.test" || exit 1
-  GOOS=windows GOARCH=amd64 go test -c -tags cit || exit 1
-  if [ -f "${suite}.test.exe" ]; then mv "${suite}.test.exe" "$outpath/${suite}64.exe" || exit 1; fi;
-  GOOS=windows GOARCH=386 go test -c -tags cit || exit 1
-  if [ -f "${suite}.test.exe" ]; then mv "${suite}.test.exe" "$outpath/${suite}32.exe" || exit 1; fi;
-  cd ..
+  go test -c -tags cit || return 1
+  ./"${suite}.test" -test.list '.*' > "$outpath/${suite}_tests.txt" || return 1
+  mv "${suite}.test" "$outpath/${suite}.amd64.test" || return 1
+  GOARCH=arm64 go test -c -tags cit || return 1
+  mv "${suite}.test" "$outpath/${suite}.arm64.test" || return 1
+  GOOS=windows GOARCH=amd64 go test -c -tags cit || return 1
+  if [ -f "${suite}.test.exe" ]; then mv "${suite}.test.exe" "$outpath/${suite}64.exe" || return 1; fi
+  GOOS=windows GOARCH=386 go test -c -tags cit || return 1
+  if [ -f "${suite}.test.exe" ]; then mv "${suite}.test.exe" "$outpath/${suite}32.exe" || return 1; fi
+}
+
+abs_outpath=$(cd "$outpath" && pwd)
+abs_root=$(cd "$imagetestroot" && pwd)
+
+# FIFO semaphore: $jobs slots in a named pipe. Each suite reads a token
+# before starting and writes one back when done.
+slotfifo=$(mktemp -u)
+mkfifo "$slotfifo"
+exec 9<>"$slotfifo"
+rm "$slotfifo"
+for i in $(seq 1 $jobs); do echo >&9; done
+
+cd "$abs_root/test_suites"
+pids=()
+names=()
+for suite in $suites; do
+  [[ -d "$suite" ]] || continue
+  read -u 9 # wait for a slot
+  (
+    build_suite "$suite" "$abs_outpath" "$abs_root"
+    rc=$?
+    echo >&9 # release slot regardless of outcome
+    exit $rc
+  ) &
+  pids+=($!)
+  names+=("$suite")
+done
+
+fail=0
+for i in "${!pids[@]}"; do
+  if ! wait "${pids[$i]}"; then
+    echo "[!] suite ${names[$i]} failed"
+    fail=1
+  fi
 done
+exec 9>&-
+[[ $fail -eq 0 ]] || exit 1