refactor(build): Update build scripts for improved functionality and clarity

Author: hunsche

infra/base-images/base-builder/precompile_honggfuzz_ubuntu_24_04

@@ -34,7 +34,7 @@ make clean
 # improve portability and `-D_HF_LINUX_NO_BFD` to remove assembly instructions
 # from the filenames.
 sed -i 's/-Werror//g' Makefile
-CC=clang CFLAGS="-O3 -funroll-loops -D_HF_LINUX_NO_BFD -Wno-unterminated-string-initialization -Wno-error" make
+CC=clang CFLAGS="-O3 -funroll-loops -D_HF_LINUX_NO_BFD -Wno-unterminated-string-initialization -Wno-error" make LDFLAGS="-lBlocksRuntime -lunwind-ptrace -lunwind-generic"
 
 # libhfuzz.a will be added by CC/CXX linker directly during linking,
 # but it's defined here to satisfy the build infrastructure

infra/base-images/base-clang/checkout_build_install_llvm_ubuntu_20_04.sh

@@ -80,7 +80,7 @@ function clone_with_retries {
 
   # Disable exit on error since we might encounter some failures while retrying.
   set +e
-  for i in $(seq 1 $CHECKOUT_RETRIES);
+  for i in $(seq 1 $CHECKOUT_RETRIES); do
     rm -rf $LOCAL_PATH
     git clone $REPOSITORY $LOCAL_PATH
     CHECKOUT_RETURN_CODE=$?

infra/base-images/base-clang/checkout_build_install_llvm_ubuntu_24_04.sh

@@ -79,8 +79,7 @@ function clone_with_retries {
 
   # Disable exit on error since we might encounter some failures while retrying.
   set +e
-  for i in $(seq 1 $CHECKOUT_RETRIES);
-  do
+  for i in $(seq 1 $CHECKOUT_RETRIES); do
     rm -rf $LOCAL_PATH
     git clone $REPOSITORY $LOCAL_PATH
     CHECKOUT_RETURN_CODE=$?
@@ -98,6 +97,16 @@ clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC
 git -C $LLVM_SRC checkout $OUR_LLVM_REVISION
 echo "Using LLVM revision: $OUR_LLVM_REVISION"
 
+# Prepare fuzz introspector.
+echo "Installing fuzz introspector"
+FUZZ_INTROSPECTOR_CHECKOUT=341ebbd72bc9116733bcfcfab5adfd7f9b633e07
+
+git clone https://github.com/ossf/fuzz-introspector.git /fuzz-introspector
+cd /fuzz-introspector
+git checkout $FUZZ_INTROSPECTOR_CHECKOUT
+git submodule init
+git submodule update
+
 # For fuzz introspector.
 echo "Applying introspector changes"
 OLD_WORKING_DIR=$PWD

infra/build/functions/build_and_push_test_images.py

@@ -56,7 +56,7 @@ def build_and_push_image(image, test_image_tag, version='legacy'):
                                                     version)
   build_image(image, [main_image_name, test_image_name], main_image_name,
               version)
-  # push_image(test_image_name)
+  push_image(test_image_name)
 
 
 def build_image(image, tags, cache_from_tag, version='latest'):
@@ -279,24 +279,23 @@ def gcb_build_and_push_images(test_image_tag: str, version_tag: str = None):
   return wait_for_build_and_report_summary(build_id)
 
 
-def build_and_push_images(test_image_tag):
+def build_and_push_images(test_image_tag, version_tag=None):
   """Builds and pushes base-images."""
   images = [
       ['base-image'],
       ['base-clang'],
       ['base-builder'],
-      ['base-runner'],
-      # Exclude 'base-builder-swift' as it takes extremely long to build because
-      # it clones LLVM.
       [
-          'base-runner-debug',
+          'base-builder-swift',
+          'base-builder-ruby',
+          'base-builder-rust',
           'base-builder-go',
           'base-builder-javascript',
           'base-builder-jvm',
           'base-builder-python',
-          'base-builder-ruby',
-          'base-builder-rust',
       ],
+      ['base-runner'],
+      ['base-runner-debug'],
   ]
   os.environ['DOCKER_BUILDKIT'] = '1'
   max_parallelization = max([len(image_list) for image_list in images])
@@ -306,7 +305,8 @@ def build_and_push_images(test_image_tag):
     for image_list in images:
       args_list = []
       for image in image_list:
-        for version in BASE_IMAGE_VERSIONS:
+        versions = [version_tag] if version_tag else BASE_IMAGE_VERSIONS
+        for version in versions:
           # Check if the specific versioned Dockerfile exists before adding.
           if version == 'legacy':
             dockerfile_path = os.path.join(IMAGES_DIR, image, 'Dockerfile')

infra/build/functions/build_lib.py

@@ -287,7 +287,7 @@ def get_signed_url(path, method='PUT', content_type=''):
   timestamp = datetime.datetime.utcnow() + datetime.timedelta(hours=1)
   timestamp = timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
 
-  path = urlparse(path)
+  path = urlparse.urlparse(path)
   blob_path = path.path.lstrip('/')
   blob = f"""{method}
 

infra/build/functions/report_generator.py

@@ -41,7 +41,7 @@ def get_visible_width(text):
 
 def _print_box(title, lines):
   """Prints a formatted box with a title and lines."""
-  box_width = 80
+  box_width = 92
   title_line = f'║ {title.center(box_width - 4)} ║'
   summary_lines = [
       '╔' + '═' * (box_width - 2) + '╗',
@@ -66,9 +66,9 @@ def generate_final_summary(all_results):
       skipped = str(data['skipped'])
       total = str(data['total'])
       line = (
-          f"  {version.ljust(15)} ► {'Passed:'.ljust(8)} {passed.ljust(2)} | "
-          f"{'Failed:'.ljust(8)} {failed.ljust(2)} | {'Skipped:'.ljust(8)} {skipped.ljust(2)} | "
-          f"{'Total:'.ljust(8)} {total.ljust(2)}")
+          f"  {version.ljust(15)} ► {'Passed:'.ljust(8)} {passed.ljust(6)} | "
+          f"{'Failed:'.ljust(8)} {failed.ljust(6)} | {'Skipped:'.ljust(8)} {skipped.ljust(6)} | "
+          f"{'Total:'.ljust(7)} {total.ljust(6)}")
       summary_lines.append(line)
 
   _print_box('FINAL BUILD REPORT', summary_lines)
@@ -85,9 +85,9 @@ def generate_comparison_table(all_results):
     print('\n✅ No projects were run.')
     return
 
-  project_col_width = 18
-  header = ' Project           |      Legacy      |   Ubuntu 20.04   |   Ubuntu 24.04'
-  separator = '-------------------+------------------+------------------+------------------'
+  project_col_width = 30
+  header = ' Project                       |      Legacy      |   Ubuntu 20.04   |   Ubuntu 24.04'
+  separator = '-------------------------------+------------------+------------------+------------------'
 
   table_lines = [header, separator]
 

infra/build/functions/trial_build.py

@@ -212,7 +212,8 @@ def trial_build_main(args=None, local_base_build=True):
   if not args.skip_build_images:
     logging.info('Starting "Build and Push Images" phase...')
     if local_base_build:
-      build_and_push_test_images.build_and_push_images(test_image_tag)
+      build_and_push_test_images.build_and_push_images(test_image_tag,
+                                                       args.version_tag)
     else:
       if not build_and_push_test_images.gcb_build_and_push_images(
           test_image_tag, version_tag=args.version_tag):
@@ -373,12 +374,15 @@ def _do_build_type_builds(args, config, credentials, build_type, projects):
       continue
 
     try:
+      tags = ['trial-build']
+      if args.branch:
+        tags.append(f'branch-{args.branch.replace("/", "-")}')
       build_ids[project_name] = (build_project.run_build(
           project_name,
           steps,
           credentials,
           build_type.type_name,
-          extra_tags=['trial-build', f'branch-{args.branch.replace("/", "-")}'],
+          extra_tags=tags,
           timeout=PROJECT_BUILD_TIMEOUT))
       time.sleep(1)  # Avoid going over 75 requests per second limit.
     except Exception as error:  # pylint: disable=broad-except
@@ -529,4 +533,4 @@ def wait_on_builds(build_ids, credentials, cloud_project, end_time,
 
 
 if __name__ == '__main__':
-  sys.exit(main())
+  sys.exit(trial_build_main())