Integrated Distributed ThinLTO (DTLTO): Design Overview

clang/docs/ThinLTO.rst

@@ -240,6 +240,39 @@ The ``BOOTSTRAP_LLVM_ENABLE_LTO=Thin`` will enable ThinLTO for stage 2 and
 stage 3 in case the compiler used for stage 1 does not support the ThinLTO
 option.
 
+Distributed ThinLTO (DTLTO)
+---------------------------
+
+DTLTO allows for the distribution of backend ThinLTO compilations via external
+distribution systems, e.g. Incredibuild. There is existing support for
+distributing ThinLTO compilations by using separate thin-link, backend
+compilation, and link steps coordinated by a build system which can handle the
+dynamic dependencies specified by the index files, such as Bazel. However, this
+often requires changes to the user's build process. With DTLTO distribution is
+managed internally in LLD as part of the traditional link step and therefore
+should be usable in any build process that can support in-process ThinLTO.
+
+DTLTO requires the LLD linker (``-fuse-ld=lld``).
+
+``-fthinlto-distributor=<path>``
+   - Specifies the ``<path>`` to the distributor process executable for DTLTO.
+   - If specified, ThinLTO backend compilations will be distributed by LLD.
+
+``-Xthinlto-distributor=<arg>``
+   - Pass ``<arg>`` to the distributor process (see ``-fthinlto-distributor=``).
+   - Can be specified multiple times to pass multiple options.
+   - Can specify multiple options by separating them with commas.
+
+Examples:
+   - ``clang -flto=thin -fthinlto-distributor=incredibuild.exe -Xthinlto-distributor=--verbose,--j10 -fuse-ld=lld``
+   - ``clang -flto=thin -fthinlto-distributor=$(which python) -Xthinlto-distributor=incredibuild.py -fuse-ld=lld``
+
+If ``-fthinlto-distributor=`` is specified Clang supplies the path to a
+distributable optimization and code generation tool to LLD. Currently this tool
+is Clang itself.
+
+See `DTLTO <https://lld.llvm.org/dtlto.html>`_ for more information.
+
 More Information
 ================
 

clang/include/clang/Driver/Options.td

@@ -969,6 +969,10 @@ def Xlinker : Separate<["-"], "Xlinker">, Flags<[LinkerInput, RenderAsInput]>,
   Visibility<[ClangOption, CLOption, FlangOption]>,
   HelpText<"Pass <arg> to the linker">, MetaVarName<"<arg>">,
   Group<Link_Group>;
+def Xthinlto_distributor_EQ : CommaJoined<["-"], "Xthinlto-distributor=">, Flags<[LinkOption]>,
+  Visibility<[ClangOption, CLOption]>,
+  HelpText<"Pass <arg> to the ThinLTO distributor">,
+  MetaVarName<"<arg>">, Group<Link_Group>;
 def Xoffload_linker : JoinedAndSeparate<["-"], "Xoffload-linker">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Pass <arg> to the offload linkers or the ones identified by -<triple>">,
@@ -4087,7 +4091,9 @@ def ffinite_loops: Flag<["-"],  "ffinite-loops">, Group<f_Group>,
 def fno_finite_loops: Flag<["-"], "fno-finite-loops">, Group<f_Group>,
   HelpText<"Do not assume that any loop is finite.">,
   Visibility<[ClangOption, CC1Option]>;
-
+def fthinlto_distributor_EQ : Joined<["-"], "fthinlto-distributor=">, Group<f_Group>,
+  HelpText<"Specifies the <path> to the distributor process executable">, MetaVarName<"<path>">,
+  Visibility<[ClangOption, CLOption]>;
 def ftrigraphs : Flag<["-"], "ftrigraphs">, Group<f_Group>,
   HelpText<"Process trigraph sequences">, Visibility<[ClangOption, CC1Option]>;
 def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group<f_Group>,

clang/lib/Driver/ToolChains/Gnu.cpp

@@ -535,6 +535,21 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                   D.getLTOMode() == LTOK_Thin);
   }
 
+  // Forward the DTLTO options to the linker. We add these unconditionally,
+  // rather than in addLTOOptions() as it is the linker that decides whether to
+  // do LTO or not dependent upon whether there are any bitcode input files in
+  // the link.
+  if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) {
+    CmdArgs.push_back(
+        Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue())));
+    CmdArgs.push_back(
+        Args.MakeArgString("--thinlto-remote-opt-tool=" +
+                           Twine(ToolChain.getDriver().getClangProgramPath())));
+
+    for (auto A : Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ))
+      CmdArgs.push_back(Args.MakeArgString("-mllvm=-thinlto-distributor-arg=" + A));
+  }
+
   if (Args.hasArg(options::OPT_Z_Xlinker__no_demangle))
     CmdArgs.push_back("--no-demangle");
 

clang/test/Driver/DTLTO/dtlto.c

@@ -0,0 +1,46 @@
+/// Check DTLTO options are forwarded to the linker.
+
+// REQUIRES: lld
+
+// RUN: echo "--target=x86_64-linux-gnu \
+// RUN:   -Xthinlto-distributor=distarg1 \
+// RUN:   -Xthinlto-distributor=distarg2,distarg3 \
+// RUN:   -fuse-ld=lld" > %t.rsp
+
+
+/// Check that options are forwarded as expected with --thinlto-distributor=.
+// RUN: %clang -### @%t.rsp -fthinlto-distributor=dist.exe %s 2>&1 | \
+// RUN:   FileCheck %s --implicit-check-not=warning
+
+// CHECK: ld.lld
+// CHECK-SAME: "--thinlto-distributor=dist.exe"
+// CHECK-SAME: "--thinlto-remote-opt-tool={{.*}}clang
+// CHECK-SAME: "-mllvm=-thinlto-distributor-arg=distarg1"
+// CHECK-SAME: "-mllvm=-thinlto-distributor-arg=distarg2"
+// CHECK-SAME: "-mllvm=-thinlto-distributor-arg=distarg3"
+
+
+/// Check that options are not added without --thinlto-distributor= and
+/// that there is an unused option warning issued for -Xthinlto-distributor= options. We
+/// specify -flto here as these options should be unaffected by it.
+// RUN: %clang -### @%t.rsp -flto=thin %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefixes=NONE,NOMORE --implicit-check-not=warning
+
+// NONE: warning: argument unused during compilation: '-Xthinlto-distributor=distarg1'
+// NONE: warning: argument unused during compilation: '-Xthinlto-distributor=distarg2,distarg3'
+// NONE:     ld.lld
+// NOMORE-NOT: --thinlto-distributor=
+// NOMORE-NOT: --thinlto-remote-opt-tool=
+// NOMORE-NOT: -mllvm
+// NOMORE-NOT: -thinlto-distributor-arg=
+
+
+/// Check the expected arguments are forwarded by default with only
+/// --thinlto-distributor=.
+// RUN: %clang --target=x86_64-linux-gnu -fthinlto-distributor=dist.exe \
+// RUN:   -fuse-ld=lld -Werror -### %s 2>&1 | \
+// RUN:   FileCheck %s --check-prefixes=DEFAULT,NOMORE
+
+// DEFAULT: ld.lld
+// DEFAULT-SAME: "--thinlto-distributor=dist.exe"
+// DEFAULT-SAME: "--thinlto-remote-opt-tool={{.*}}clang

clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp

@@ -645,7 +645,7 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
           std::make_unique<raw_fd_ostream>(FD, true));
     };
 
-    if (Error Err = LTOBackend.run(AddStream))
+    if (Error Err = LTOBackend.run(AddStream, /*AddBuffer=*/nullptr))
       return Err;
 
     if (Args.hasArg(OPT_lto_emit_llvm) || Args.hasArg(OPT_lto_emit_asm))

cross-project-tests/CMakeLists.txt

@@ -19,11 +19,15 @@ set(CROSS_PROJECT_TEST_DEPS
   FileCheck
   check-gdb-llvm-support
   count
-  llvm-dwarfdump
+  llvm-ar
   llvm-config
+  llvm-dwarfdump
+  llvm-lto2
   llvm-objdump
-  split-file
+  llvm-profdata
   not
+  opt
+  split-file
   )
 
 if ("clang" IN_LIST LLVM_ENABLE_PROJECTS)
@@ -94,6 +98,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}

cross-project-tests/dtlto/README.md

@@ -0,0 +1,3 @@
+Tests for DTLTO (integrated distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.

cross-project-tests/dtlto/archive-thin.test

@@ -0,0 +1,72 @@
+## Simple test that a DTLTO link succeeds and outputs the expected set of files
+## correctly when thin archives are present.
+
+# RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir
+# RUN: %clang --target=x86_64-linux-gnu -c foo.c -o foo.o
+# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin bar.c -o bar.o
+# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin dog.c -o dog.o
+# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin cat.c -o cat.o
+# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin _start.c -o _start.o
+
+# RUN: llvm-ar rcs foo.a foo.o --thin
+## Create this bitcode thin archive in a sub-directory to test the expansion of
+## the path to a bitcode file which is referenced using "..", e.g. in this case
+## "../bar.o". The ".." should be collapsed in any expansion to avoid
+## referencing an unknown directory on the remote side.
+# RUN: mkdir lib
+# RUN: llvm-ar rcs lib/bar.a bar.o --thin
+## Create this bitcode thin archive with an absolute path entry containing "..".
+# RUN: llvm-ar rcs dog.a %t.dir/lib/../dog.o --thin
+# RUN: llvm-ar rcs cat.a cat.o --thin
+# RUN: llvm-ar rcs _start.a _start.o --thin
+
+# RUN: mkdir %t.dir/out && cd %t.dir/out
+
+# RUN: %clang --target=x86_64-linux-gnu \
+# RUN:   %t.dir/foo.a %t.dir/lib/bar.a ../_start.a %t.dir/cat.a -Wl,--whole-archive,../dog.a \
+# RUN:   -flto=thin \
+# RUN:   -fthinlto-distributor=%python \
+# RUN:   -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py \
+# RUN:   --save-temps \
+# RUN:   -fuse-ld=lld \
+# RUN:   -nostdlib \
+# RUN:   -nostartfiles \
+# RUN:   -Wl,--save-temps \
+# RUN:   -Wl,-mllvm,--thinlto-remote-opt-tool-arg=-save-temps=cwd \
+# RUN:   -Werror
+
+## Check that the required output files have been created.
+# RUN: ls | FileCheck %s --check-prefix=OUTPUTS \
+# RUN:     --implicit-check-not=cat --implicit-check-not=foo
+
+## The DTLTO backend emits the JSON jobs description and summary shards.
+# OUTPUTS-DAG: a.{{[0-9]+}}.dist-file.json
+# OUTPUTS-DAG: bar.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}}
+# OUTPUTS-DAG: dog.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}}
+# OUTPUTS-DAG: _start.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}}
+## Native output object files.
+# OUTPUTS-DAG: bar.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}}
+# OUTPUTS-DAG: dog.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}}
+# OUTPUTS-DAG: _start.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}}
+
+## Check that bar.o and dog.o are not referenced using "..".
+# RUN: not grep '\.\.\(/\|\\\\\)\(bar\|dog\)\.o' a.*.dist-file.json
+
+#--- foo.c
+__attribute__((retain)) void foo() {}
+
+#--- bar.c
+extern void foo();
+__attribute__((retain)) void bar() { foo(); }
+
+#--- dog.c
+__attribute__((retain)) void dog() {}
+
+#--- cat.c
+__attribute__((retain)) void cat() {}
+
+#--- _start.c
+extern void bar();
+__attribute__((retain)) void _start() {
+  bar();
+}

cross-project-tests/dtlto/dtlto-translate-options.ll

@@ -0,0 +1,144 @@
+;; Check that the expected Clang arguments are generated by DTLTO for the 
+;; backend compilations and are accepted by Clang.
+
+; RUN: rm -rf %t && split-file %s %t && cd %t
+
+;; Generate bitcode files with a summary index.
+; RUN: opt -thinlto-bc x86_64-unknown-linux-gnu.ll -o x86_64-unknown-linux-gnu.bc
+; RUN: opt -thinlto-bc x86_64-pc-windows-msvc.ll   -o x86_64-pc-windows-msvc.bc
+
+
+;; Check that any invalid arguments would cause a Clang error. This property is
+;; relied on by the actual testcases later in this test.
+; RUN: not %clang -x ir x86_64-unknown-linux-gnu.ll \
+; RUN:     -invalid-incorrect-not-an-option 2>&1 | FileCheck %s --check-prefix=SANITY1
+; SANITY1: unknown argument: '-invalid-incorrect-not-an-option'
+
+
+;; Define a substitution used to simplify the testcases.
+; DEFINE: %{distributor} = dummy
+; DEFINE: %{extra_flags} = dummy
+; DEFINE: %{triple} = dummy
+; DEFINE: %{command} = llvm-lto2 run \
+; DEFINE:   -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/%{distributor} \
+; DEFINE:   -thinlto-remote-opt-tool-arg=-Wunused-command-line-argument \
+; DEFINE:   @%{triple}.rsp %{extra_flags}
+
+
+;; Write common arguments to a response files.
+
+; RUN: echo "x86_64-unknown-linux-gnu.bc -o x86_64-unknown-linux-gnu.o \
+; RUN:       -dtlto \
+; RUN:       -dtlto-remote-opt-tool=%clang \
+; RUN:       -thinlto-remote-opt-tool-arg=-Werror \
+; RUN:       -dtlto-distributor=%python \
+; RUN:       -r=x86_64-unknown-linux-gnu.bc,globalfunc1,plx" > x86_64-unknown-linux-gnu.rsp
+
+; RUN: echo "x86_64-pc-windows-msvc.bc -o x86_64-pc-windows-msvc.o \
+; RUN:       -dtlto \
+; RUN:       -dtlto-remote-opt-tool=%clang \
+; RUN:       -thinlto-remote-opt-tool-arg=-Werror \
+; RUN:       -thinlto-remote-opt-tool-arg=-Wno-override-module \
+; RUN:       -dtlto-distributor=%python \
+; RUN:       -r=x86_64-pc-windows-msvc.bc,globalfunc2,plx" > x86_64-pc-windows-msvc.rsp
+
+
+;; Check that boolean configuration states are translated as expected and Clang
+;; accepts them.
+
+; RUN: echo " \
+; RUN:   --addrsig=1 \
+; RUN:   -function-sections=1 \
+; RUN:   -data-sections=1" > on.rsp
+
+; RUN: echo " \
+; RUN:   --addrsig=0 \
+; RUN:   -function-sections=0 \
+; RUN:   -data-sections=0" > off.rsp
+
+;; Perform DTLTO with configuration state set.
+; REDEFINE: %{extra_flags} = @on.rsp
+; REDEFINE: %{distributor} = local.py
+; REDEFINE: %{triple} = x86_64-unknown-linux-gnu
+; RUN: %{command}
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=ON \
+; RUN:     --implicit-check-not=-no-pgo-warn-mismatch
+; ON-DAG: "-faddrsig"
+; ON-DAG: "-ffunction-sections"
+; ON-DAG: "-fdata-sections"
+
+;; Perform DTLTO with configuration state unset.
+; REDEFINE: %{extra_flags} = @off.rsp
+; REDEFINE: %{distributor} = local.py
+; RUN: %{command}
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=OFF
+; OFF-NOT: --implicit-check-not=--faddrsig
+; OFF-NOT: --implicit-check-not=--ffunction-sections
+; OFF-NOT: --implicit-check-not=--fdata-sections
+; OFF-NOT: --implicit-check-not=-no-pgo-warn-mismatch
+
+
+;; Check optimisation level.
+
+; RUN: llvm-lto2 run \
+; RUN:   -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+; RUN:   @x86_64-unknown-linux-gnu.rsp \
+; RUN:   -O3
+
+; RUN: not llvm-lto2 run \
+; RUN:   -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
+; RUN:   @x86_64-unknown-linux-gnu.rsp \
+; RUN:   -O3 2>&1 | FileCheck %s --check-prefix=OPTLEVEL
+; OPTLEVEL-DAG: "-O3"
+
+
+;; Check relocation model.
+
+; REDEFINE: %{extra_flags} = -relocation-model=pic
+; REDEFINE: %{distributor} = local.py
+; RUN: %{command}
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=PIC
+; PIC: -fpic
+
+
+; REDEFINE: %{extra_flags} = -relocation-model=pic
+; REDEFINE: %{distributor} = local.py
+; REDEFINE: %{triple} = x86_64-pc-windows-msvc
+; RUN: %{command}
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=NOPIC
+; REDEFINE: %{triple} = x86_64-unknown-linux-gnu
+; NOPIC-NOT: -fpic
+
+;; Check specifying a sample profile.
+; REDEFINE: %{extra_flags} = --lto-sample-profile-file="missing.profdata"
+; REDEFINE: %{distributor} = local.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE_ERR
+; SAMPLE_PROFILE_ERR: no such file or directory: 'missing.profdata'
+; REDEFINE: %{distributor} = validate.py
+; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE
+; SAMPLE_PROFILE-DAG: "-fprofile-sample-use=missing.profdata"
+
+
+;--- x86_64-unknown-linux-gnu.ll
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @globalfunc1() {
+entry:
+  ret void
+}
+
+;--- x86_64-pc-windows-msvc.ll
+
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+define void @globalfunc2() {
+entry:
+  ret void
+}