tc-build: Introduce '--multicall'

LLVM introduced a multicall binary feature (similar to busybox) where
all tools are linked into a single binary then symlinks to individual
tools switch the behavior of the binary:

  $ ls -l clang llvm ld.lld
  lrwxrwxrwx    - nathan  4 Feb 16:03 clang -> llvm
  lrwxrwxrwx    - nathan  4 Feb 16:03 ld.lld -> llvm
  .rwxr-xr-x 288M nathan  4 Feb 16:03 llvm

  $ ./clang --version | head -1
  ClangBuiltLinux clang version 23.0.0git (https://github.com/llvm/llvm-project.git dcf853df8fda885cafbc02f916ef01684b7d1104)

  $ ./llvm clang --version | head -1
  ClangBuiltLinux clang version 23.0.0git (https://github.com/llvm/llvm-project.git dcf853df8fda885cafbc02f916ef01684b7d1104)

  $ ./ld.lld --version | head -1
  ClangBuiltLinux LLD 23.0.0 (https://github.com/llvm/llvm-project.git dcf853df8fda885cafbc02f916ef01684b7d1104) (compatible with GNU linkers)

  $ ./llvm ld.lld --version | head -1
  ClangBuiltLinux LLD 23.0.0 (https://github.com/llvm/llvm-project.git dcf853df8fda885cafbc02f916ef01684b7d1104) (compatible with GNU linkers)

This combines the speed benefits of static linking with the space
savings of dynamic linking.

  $ diskus pgo-bolt
  632.09 MB (632,090,624 bytes)

  $ diskus multicall-pgo-bolt
  321.14 MB (321,138,688 bytes)

As this changes the layout of the installation, this may result in
functional differences depending on software assumptions, so require the
user to opt into this with a '--multicall' option. Adjust the BOLT
process to use the correct binary.

Signed-off-by: Nathan Chancellor <nathan@kernel.org>

Author: nathanchance

build-llvm.py

@@ -236,6 +236,14 @@
                     '''),
                     type=str,
                     choices=['thin', 'full'])
+parser.add_argument('-m',
+                    '--multicall',
+                    help=textwrap.dedent('''\
+                    Build LLVM as a multicall binary via the LLVM_TOOL_LLVM_DRIVER_BUILD CMake option.
+                    This results in a much smaller installation on disk.
+
+                    '''),
+                    action='store_true')
 parser.add_argument('-n',
                     '--no-update',
                     help=textwrap.dedent('''\
@@ -536,6 +544,8 @@
 if args.vendor_string:
     common_cmake_defines['CLANG_VENDOR'] = args.vendor_string
     common_cmake_defines['LLD_VENDOR'] = args.vendor_string
+if args.multicall:
+    common_cmake_defines['LLVM_TOOL_LLVM_DRIVER_BUILD'] = 'ON'
 if args.defines:
     defines = dict(define.split('=', 1) for define in args.defines)
     common_cmake_defines.update(defines)

tc_build/llvm.py

@@ -74,26 +74,27 @@ def bolt_clang(self):
 
         tc_build.utils.print_header(f"Performing BOLT with {mode}")
 
-        # clang-#: original binary
-        # clang.bolt: BOLT optimized binary
+        # real_binary: clang-# or llvm (if multicall is enabled)
+        # bolt_binary: clang.bolt or llvm.bolt
         # .bolt will become original binary after optimization
-        clang = Path(self.folders.build, 'bin/clang').resolve()
-        clang_bolt = clang.with_name('clang.bolt')
+        real_binary = Path(self.folders.build, 'bin/clang').resolve()
+        binary_prefix = 'llvm' if self.multicall_is_enabled() else 'clang'
+        bolt_binary = real_binary.with_name(f"{binary_prefix}.bolt")
 
-        bolt_profile = Path(self.folders.build, 'clang.fdata')
+        bolt_profile = Path(self.folders.build, f"{binary_prefix}.fdata")
 
         if mode == 'instrumentation':
-            # clang.inst: instrumented binary, will be removed after generating profiles
-            clang_inst = clang.with_name('clang.inst')
+            # clang.inst / llvm.inst: instrumented binary, will be removed after generating profiles
+            inst_binary = real_binary.with_name(f"{binary_prefix}.inst")
 
             clang_inst_cmd = [
                 self.tools.llvm_bolt,
                 '--instrument',
                 f"--instrumentation-file={bolt_profile}",
                 '--instrumentation-file-append-pid',
                 '-o',
-                clang_inst,
-                clang,
+                inst_binary,
+                real_binary,
             ]
             # When running an instrumented binary on certain platforms (namely
             # Apple Silicon), there may be hangs due to instrumentation in
@@ -104,7 +105,16 @@ def bolt_clang(self):
                 clang_inst_cmd.append('--conservative-instrumentation')
             self.run_cmd(clang_inst_cmd)
 
-            self.bolt_builder.bolt_instrumentation = True
+            if binary_prefix == 'llvm':
+                # The multicall tools are all symlinked to the 'llvm' binary.
+                # To avoid having to mess with those symlinks...
+                orig_binary = real_binary.with_name('llvm.orig')
+                real_binary.replace(orig_binary)  # mv llvm llvm.orig
+                inst_binary.replace(real_binary)  # mv llvm.inst llvm
+            else:
+                # This option changes CC when building Linux, which is only
+                # needed when just clang is instrumented.
+                self.bolt_builder.bolt_instrumentation = True
 
         if mode == 'sampling':
             self.bolt_builder.bolt_sampling_output = Path(self.folders.build, 'perf.data')
@@ -115,6 +125,11 @@ def bolt_clang(self):
         # With instrumentation, we need to combine the profiles we generated,
         # as they are separated by PID
         if mode == 'instrumentation':
+            # Undo shuffle from above
+            if binary_prefix == 'llvm':
+                real_binary.replace(inst_binary)  # mv llvm llvm.inst
+                orig_binary.replace(real_binary)  # mv llvm.orig llvm
+
             fdata_files = bolt_profile.parent.glob(f"{bolt_profile.name}.*.fdata")
 
             # merge-fdata will print one line for each .fdata it merges.
@@ -139,7 +154,7 @@ def bolt_clang(self):
                 self.bolt_builder.bolt_sampling_output,
                 '-o',
                 bolt_profile,
-                clang,
+                real_binary,
             ]
             self.run_cmd(perf2bolt_cmd)
             self.bolt_builder.bolt_sampling_output.unlink()
@@ -166,18 +181,18 @@ def bolt_clang(self):
             '--dyno-stats',
             f"--icf={icf_val}",
             '-o',
-            clang_bolt,
+            bolt_binary,
             f"--reorder-blocks={'cache+' if use_cache_plus else 'ext-tsp'}",
             f"--reorder-functions={reorder_funcs_val}",
             '--split-all-cold',
             f"--split-functions{'=3' if use_sf_val else ''}",
             '--use-gnu-stack',
-            clang,
+            real_binary,
         ]
         self.run_cmd(clang_opt_cmd)
-        clang_bolt.replace(clang)
+        bolt_binary.replace(real_binary)
         if mode == 'instrumentation':
-            clang_inst.unlink()
+            inst_binary.unlink()
 
     def build(self):
         if not self.folders.build:
@@ -388,6 +403,9 @@ def host_target(self):
     def host_target_is_enabled(self):
         return 'all' in self.targets or self.host_target() in self.targets
 
+    def multicall_is_enabled(self):
+        return self.cmake_defines.get('LLVM_TOOL_LLVM_DRIVER_BUILD', 'OFF') == 'ON'
+
     def project_is_enabled(self, project):
         return 'all' in self.projects or project in self.projects
 
@@ -573,7 +591,6 @@ def __init__(self):
 
         self.cmake_defines['LLVM_BUILD_INSTRUMENTED'] = 'IR'
         self.cmake_defines['LLVM_BUILD_RUNTIME'] = 'OFF'
-        self.cmake_defines['LLVM_LINK_LLVM_DYLIB'] = 'ON'
 
     def configure(self):
         # The following defines are needed to avoid thousands of warnings
@@ -605,6 +622,13 @@ def configure(self):
             self.cmake_defines['CMAKE_C_FLAGS'] = ' '.join(cflags)
             self.cmake_defines['CMAKE_CXX_FLAGS'] = ' '.join(cxxflags)
 
+        # These are currently incompatible:
+        # https://github.com/llvm/llvm-project/pull/133596
+        # But that should not matter much in this case because multicall uses
+        # much less disk space.
+        if not self.multicall_is_enabled():
+            self.cmake_defines['LLVM_LINK_LLVM_DYLIB'] = 'ON'
+
         super().configure()
 
     def generate_profdata(self):