[ci] Overcommit number of CPUs when the machine allows it

.github/workflows/root-ci-config/build_root.py

@@ -13,28 +13,30 @@
    explicitly appended to the shell log.
       e.g. `os.chdir(x)` requires `cd x` to be appended to the shell log  """
 
 import argparse
 import datetime
 import os
 import platform
 import shutil
 import subprocess
 import sys
 import tarfile
 import time
 
 import build_utils
 import openstack
 from build_utils import (
     calc_options_hash,
     die,
     github_log_group,
     is_macos,
+    is_windows,
     load_config,
     subprocess_with_capture,
     subprocess_with_log,
     upload_file,
+    get_cpu_count,
 )
 
 S3CONTAINER = 'ROOT-build-artifacts'  # Used for uploads
 S3URL = 'https://s3.cern.ch/swift/v1/' + S3CONTAINER  # Used for downloads
@@ -45,9 +47,8 @@
     print("Failed to open the S3 connection:", exc, file=sys.stderr)
     CONNECTION = None
 
-WINDOWS = (os.name == 'nt')
-WORKDIR = (os.environ['HOME'] + '/ROOT-CI') if not WINDOWS else 'C:/ROOT-CI'
-COMPRESSIONLEVEL = 6 if not WINDOWS else 1
+WORKDIR = (os.environ['HOME'] + '/ROOT-CI') if not is_windows() else 'C:/ROOT-CI'
+COMPRESSIONLEVEL = 6 if not is_windows() else 1
 
 
 def main():
@@ -78,7 +79,7 @@
 
     options = build_utils.cmake_options_from_dict(options_dict)
 
-    if WINDOWS:
+    if is_windows():
         options = "-Thost=x64 " + options
 
         if args.architecture == 'x86':
@@ -123,7 +124,7 @@
 
     testing: bool = options_dict['testing'].lower() == "on"
 
-    if not WINDOWS:
+    if not is_windows():
         show_node_state()
 
     build(options, args.buildtype)
@@ -140,7 +141,7 @@
 
     if testing:
         extra_ctest_flags = ""
-        if WINDOWS:
+        if is_windows():
             extra_ctest_flags += "--repeat until-pass:5 "
             extra_ctest_flags += "--build-config " + args.buildtype
 
@@ -210,7 +211,7 @@
     if WORKDIR in ("", "/"):
         die(1, "WORKDIR not set")
 
-    if WINDOWS:
+    if is_windows():
         # windows
         os.environ['COMSPEC'] = 'powershell.exe'
         result = subprocess_with_log(f"""
@@ -374,8 +375,8 @@
 
 @github_log_group("Build")
 def cmake_build(buildtype):
-    generator_flags = "-- '-verbosity:minimal'" if WINDOWS else ""
-    parallel_jobs = "4" if WINDOWS else str(os.cpu_count())
+    generator_flags = "-- '-verbosity:minimal'" if is_windows() else ""
+    parallel_jobs = "4" if is_windows() else str(get_cpu_count())
 
     builddir = os.path.join(WORKDIR, "build")
     result = subprocess_with_log(f"""

.github/workflows/root-ci-config/build_utils.py

@@ -1,19 +1,20 @@
 #!/usr/bin/env false
 
 import json
 import os
 import subprocess
 import sys
 import textwrap
 import datetime
 import time
 import platform
+import math
 from functools import wraps
 from hashlib import sha1
 from http import HTTPStatus
 from shutil import which
 from typing import Callable, Dict
 from collections import namedtuple
 
 from openstack.connection import Connection
 from requests import get
@@ -21,6 +22,9 @@
 def is_macos():
     return 'Darwin' == platform.system()
 
+def is_windows():
+    return os.name == 'nt'
+
 class SimpleTimer:
     def __init__(self):
         self._start_time = time.perf_counter()
@@ -218,7 +222,7 @@
     """
     options_and_defines = options
     if ('march=native' in options):
         print_info(f"A march=native build was detected.")
         compiler_name = 'c++' if which('c++') else 'clang++'
         command = f'echo | {compiler_name} -dM -E - -march=native'
         sp_result = subprocess.run([command], shell=True, capture_output=True, text=True)
@@ -255,12 +259,12 @@
         try:
             create_object_local()
             success = True
         except:
             success = False
             sleep_time = sleep_time_unit * attempt
             print_warning(f"""Attempt {attempt} to upload {src_file} to {dest_object} failed. Retrying in {sleep_time} seconds...""")
             time.sleep(sleep_time)
         if success: break
 
     # We try one last time
     create_object_local()
@@ -302,3 +306,12 @@
         log.add(f"\ncurl --output {destination}/artifacts.tar.gz {url}/{latest}\n")
 
     return f"{destination}/artifacts.tar.gz"
+
+def get_cpu_count():
+    base_cpu_count = os.cpu_count()
+    cpu_count = base_cpu_count
+    if not is_windows() and not is_macos():
+        cpu_count = math.ceil(base_cpu_count * 1.15)
+        print_info(f"Hardware CPU count is {base_cpu_count}: since this is Linux, we overcommit the node running on {cpu_count} CPUs.")
+
+    return cpu_count