diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 0b2a88a4a6..335ba17f69 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -205,11 +205,15 @@ def _rust_library_common(ctx, crate_type): crate_type, disable_pipelining = getattr(ctx.attr, "disable_pipelining", False), ): + # The hollow rlib uses .rlib extension (not .rmeta) so rustc reads it as an + # rlib archive containing lib.rmeta with optimized MIR. It is placed in a + # "_hollow/" subdirectory so the full rlib and hollow rlib never appear in the + # same -Ldependency= search directory (which would cause E0463). rust_metadata = ctx.actions.declare_file( - paths.replace_extension(rust_lib_name, ".rmeta"), - sibling = rust_lib, + "_hollow/" + rust_lib_name[:-len(".rlib")] + "-hollow.rlib", ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) + metadata_supports_pipelining = ( can_use_metadata_for_pipelining(toolchain, crate_type) and not ctx.attr.disable_pipelining diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 42616d8868..348b62aaa9 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -663,7 +663,7 @@ def _disambiguate_libs(actions, toolchain, crate_info, dep_info, use_pic): visited_libs[name] = artifact return ambiguous_libs -def _depend_on_metadata(crate_info, force_depend_on_objects): +def _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link = False): """Determines if we can depend on metadata for this crate. By default (when pipelining is disabled or when the crate type needs to link against @@ -673,9 +673,22 @@ def _depend_on_metadata(crate_info, force_depend_on_objects): In some rare cases, even if both of those conditions are true, we still want to depend on objects. This is what force_depend_on_objects is. + When experimental_use_cc_common_link is True, bin/cdylib crates also use hollow + rlib deps. The rustc step only emits .o files (no rustc linking), so SVH chain + consistency is sufficient; the actual linking is done by cc_common.link, which + does not check SVH. + + Callers are responsible for zeroing out experimental_use_cc_common_link for + exec-platform builds before calling this function (see rustc_compile_action). + Exec-platform binaries (build scripts) must use full rlib deps because their + CcInfo linking contexts may lack a CC toolchain. + Args: crate_info (CrateInfo): The Crate to determine this for. force_depend_on_objects (bool): if set we will not depend on metadata. + experimental_use_cc_common_link (bool): if set, bin/cdylib crates also use + hollow rlib deps for SVH consistency. Must already be False for + exec-platform builds when this function is called. Returns: Whether we can depend on metadata for this crate. @@ -683,6 +696,11 @@ def _depend_on_metadata(crate_info, force_depend_on_objects): if force_depend_on_objects: return False + if experimental_use_cc_common_link and crate_info.type in ("bin", "cdylib"): + # cc_common.link: rustc only emits .o files, so hollow rlib deps are safe and + # keep the SVH chain consistent (avoiding E0460 from nondeterministic proc macros). + return True + return crate_info.type in ("rlib", "lib") def collect_inputs( @@ -770,7 +788,7 @@ def collect_inputs( linkstamp_outs = [] transitive_crate_outputs = dep_info.transitive_crate_outputs - if _depend_on_metadata(crate_info, force_depend_on_objects): + if _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link): transitive_crate_outputs = dep_info.transitive_metadata_outputs nolinkstamp_compile_direct_inputs = [] @@ -806,6 +824,12 @@ def collect_inputs( transitive = [ crate_info.srcs, transitive_crate_outputs, + # Always include hollow rlibs so they are present in the sandbox for + # -Ldependency= resolution. Binaries and proc-macros compile against full + # rlib --extern deps but need hollow rlibs available for transitive + # dependency resolution when those rlibs were themselves compiled against + # hollow deps. For rlib/lib crates this is a no-op (already included above). + dep_info.transitive_metadata_outputs, crate_info.compile_data, dep_info.transitive_proc_macro_data, toolchain.all_files, @@ -909,6 +933,7 @@ def construct_arguments( use_json_output = False, build_metadata = False, force_depend_on_objects = False, + experimental_use_cc_common_link = False, skip_expanding_rustc_env = False, require_explicit_unstable_features = False, error_format = None): @@ -1044,8 +1069,14 @@ def construct_arguments( error_format = "json" if build_metadata: - # Configure process_wrapper to terminate rustc when metadata are emitted - process_wrapper_flags.add("--rustc-quit-on-rmeta", "true") + if crate_info.type in ("rlib", "lib"): + # Hollow rlib approach (Buck2-style): rustc runs to completion with -Zno-codegen, + # producing a hollow .rlib (metadata only, no object code) via --emit=link=. + # No need to kill rustc — -Zno-codegen skips codegen entirely and exits quickly. + rustc_flags.add("-Zno-codegen") + + # else: IDE-only metadata for non-rlib types (bin, proc-macro, etc.): rustc exits + # naturally after writing .rmeta via --emit=dep-info,metadata (no kill needed). if crate_info.rustc_rmeta_output: process_wrapper_flags.add("--output-file", crate_info.rustc_rmeta_output.path) elif crate_info.rustc_output: @@ -1078,7 +1109,14 @@ def construct_arguments( emit_without_paths = [] for kind in emit: - if kind == "link" and crate_info.type == "bin" and crate_info.output != None: + if kind == "link" and build_metadata and crate_info.type in ("rlib", "lib") and crate_info.metadata: + # Hollow rlib: direct rustc's link output to the -hollow.rlib path. + # The file has .rlib extension so rustc reads it as an rlib archive + # (with optimized MIR in lib.rmeta). Using a .rmeta path would cause + # E0786 "found invalid metadata files" because rustc parses .rmeta files + # as raw metadata blobs, not rlib archives. + rustc_flags.add(crate_info.metadata, format = "--emit=link=%s") + elif kind == "link" and crate_info.type == "bin" and crate_info.output != None: rustc_flags.add(crate_info.output, format = "--emit=link=%s") else: emit_without_paths.append(kind) @@ -1153,7 +1191,7 @@ def construct_arguments( include_link_flags = include_link_flags, ) - use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects) + use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link) # These always need to be added, even if not linking this crate. add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct, use_metadata) @@ -1322,6 +1360,13 @@ def rustc_compile_action( rustc_output = crate_info.rustc_output rustc_rmeta_output = crate_info.rustc_rmeta_output + # Use the hollow rlib approach (Buck2-style) for rlib/lib crate types when a metadata + # action is being created. This always applies for rlib/lib regardless of whether + # pipelining is globally enabled — the hollow rlib is simpler than killing rustc. + # Non-rlib types (bin, proc-macro, etc.) use --emit=dep-info,metadata instead + # (rustc exits naturally after writing .rmeta, no process-wrapper kill needed). + use_hollow_rlib = bool(build_metadata) and crate_info.type in ("rlib", "lib") + # Determine whether to use cc_common.link: # * either if experimental_use_cc_common_link is 1, # * or if experimental_use_cc_common_link is -1 and @@ -1335,6 +1380,12 @@ def rustc_compile_action( elif ctx.attr.experimental_use_cc_common_link == -1: experimental_use_cc_common_link = toolchain._experimental_use_cc_common_link + # Exec-platform binaries (build scripts) skip cc_common.link: exec-configuration + # rlib deps may lack a CC toolchain, causing empty CcInfo linking contexts. They + # use standard rustc linking with full rlib deps instead. + if experimental_use_cc_common_link and is_exec_configuration(ctx): + experimental_use_cc_common_link = False + dep_info, build_info, linkstamps = collect_deps( deps = deps, proc_macro_deps = proc_macro_deps, @@ -1376,17 +1427,34 @@ def rustc_compile_action( experimental_use_cc_common_link = experimental_use_cc_common_link, ) - # The types of rustc outputs to emit. - # If we build metadata, we need to keep the command line of the two invocations - # (rlib and rmeta) as similar as possible, otherwise rustc rejects the rmeta as - # a candidate. - # Because of that we need to add emit=metadata to both the rlib and rmeta invocation. - # - # When cc_common linking is enabled, emit a `.o` file, which is later - # passed to the cc_common.link action. + # The main Rustc action uses FULL rlib deps so the full rlib it produces records + # full-rlib SVHs. A downstream binary links against full rlibs; if the Rustc action + # had used hollow rlib deps instead, nondeterministic proc macros could produce + # different SVHs for the hollow vs full rlib, causing E0460 in the binary build. + # The RustcMetadata action still uses hollow rlibs (compile_inputs_for_metadata) + # so it can start before full codegen of its deps completes. + compile_inputs_for_metadata = compile_inputs + if use_hollow_rlib: + compile_inputs, _, _, _, _, _ = collect_inputs( + ctx = ctx, + file = ctx.file, + files = ctx.files, + linkstamps = linkstamps, + toolchain = toolchain, + cc_toolchain = cc_toolchain, + feature_configuration = feature_configuration, + crate_info = crate_info, + dep_info = dep_info, + build_info = build_info, + lint_files = lint_files, + stamp = stamp, + force_depend_on_objects = True, + experimental_use_cc_common_link = experimental_use_cc_common_link, + ) + + # The main Rustc action emits dep-info and link (the full rlib/binary/cdylib). + # When cc_common linking is enabled, emit a `.o` file instead. emit = ["dep-info", "link"] - if build_metadata: - emit.append("metadata") if experimental_use_cc_common_link: emit = ["obj"] @@ -1421,12 +1489,24 @@ def rustc_compile_action( force_all_deps_direct = force_all_deps_direct, stamp = stamp, use_json_output = bool(build_metadata) or bool(rustc_output) or bool(rustc_rmeta_output), + # Force full rlib --extern deps so the full rlib records full-rlib SVHs. + force_depend_on_objects = use_hollow_rlib, + experimental_use_cc_common_link = experimental_use_cc_common_link, skip_expanding_rustc_env = skip_expanding_rustc_env, require_explicit_unstable_features = require_explicit_unstable_features, ) args_metadata = None if build_metadata: + if use_hollow_rlib: + # Hollow rlib: emit dep-info and link (directed to the -hollow.rlib path via + # -Zno-codegen). dep-info must be included: it affects the SVH stored in the + # rlib, so both actions must include it to keep SVHs consistent. + metadata_emit = ["dep-info", "link"] + else: + # IDE-only metadata for non-rlib types (bin, proc-macro, etc.): rustc exits + # naturally after writing .rmeta with --emit=dep-info,metadata. + metadata_emit = ["dep-info", "metadata"] args_metadata, _ = construct_arguments( ctx = ctx, attr = attr, @@ -1434,7 +1514,7 @@ def rustc_compile_action( toolchain = toolchain, tool_path = toolchain.rustc.path, cc_toolchain = cc_toolchain, - emit = emit, + emit = metadata_emit, feature_configuration = feature_configuration, crate_info = crate_info, dep_info = dep_info, @@ -1449,6 +1529,7 @@ def rustc_compile_action( stamp = stamp, use_json_output = True, build_metadata = True, + experimental_use_cc_common_link = experimental_use_cc_common_link, require_explicit_unstable_features = require_explicit_unstable_features, ) @@ -1457,6 +1538,13 @@ def rustc_compile_action( # this is the final list of env vars env.update(env_from_args) + if use_hollow_rlib: + # Both the metadata action and the full Rustc action must have RUSTC_BOOTSTRAP=1 + # for SVH compatibility. RUSTC_BOOTSTRAP=1 changes the crate hash — setting it + # on only one action would cause SVH mismatch even for deterministic crates. + # This enables -Zno-codegen on stable Rust compilers for the metadata action. + env["RUSTC_BOOTSTRAP"] = "1" + if hasattr(attr, "version") and attr.version != "0.0.0": formatted_version = " v{}".format(attr.version) else: @@ -1525,7 +1613,7 @@ def rustc_compile_action( if args_metadata: ctx.actions.run( executable = ctx.executable._process_wrapper, - inputs = compile_inputs, + inputs = compile_inputs_for_metadata, outputs = [build_metadata] + [x for x in [rustc_rmeta_output] if x], env = env, arguments = args_metadata.all, @@ -2144,9 +2232,14 @@ def add_crate_link_flags(args, dep_info, force_all_deps_direct = False, use_meta crate_to_link_flags = _crate_to_link_flag_metadata if use_metadata else _crate_to_link_flag args.add_all(direct_crates, uniquify = True, map_each = crate_to_link_flags) + # Use hollow rlib directories for -Ldependency= when use_metadata=True (rlib/lib) + # so that both --extern= and -Ldependency= point to the same hollow rlib file. + # When use_metadata=False (bins, proc-macros), use full rlib directories; pointing + # to hollow dirs alongside full --extern= args would cause E0463 (ambiguous crate). + get_dirname = _get_crate_dirname_pipelined if use_metadata else _get_crate_dirname args.add_all( dep_info.transitive_crates, - map_each = _get_crate_dirname, + map_each = get_dirname, uniquify = True, format_each = "-Ldependency=%s", ) @@ -2204,6 +2297,24 @@ def _get_crate_dirname(crate): """ return crate.output.dirname +def _get_crate_dirname_pipelined(crate): + """For pipelined compilation: returns the _hollow/ directory for pipelined crates + + When a crate supports pipelining and has a hollow rlib in its _hollow/ subdirectory, + pointing -Ldependency= to that subdirectory lets rustc find the hollow rlib (which has + the correct SVH matching downstream metadata). Pointing to the parent directory instead + would expose the full rlib (compiled separately, with a different SVH), causing E0460. + + Args: + crate (CrateInfo): A CrateInfo provider from the current rule + + Returns: + str: The directory to use for -Ldependency= search. + """ + if crate.metadata and crate.metadata_supports_pipelining: + return crate.metadata.dirname + return crate.output.dirname + def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows = False, for_darwin = False, flavor_msvc = False): artifact = get_preferred_artifact(lib, use_pic) if ambiguous_libs and artifact.path in ambiguous_libs: diff --git a/rust/settings/settings.bzl b/rust/settings/settings.bzl index aaed525204..70f92a97fe 100644 --- a/rust/settings/settings.bzl +++ b/rust/settings/settings.bzl @@ -112,10 +112,18 @@ def use_real_import_macro(): ) def pipelined_compilation(): - """When set, this flag causes rustc to emit `*.rmeta` files and use them for `rlib -> rlib` dependencies. + """When set, this flag enables pipelined compilation for rlib/lib crates. - While this involves one extra (short) rustc invocation to build the rmeta file, - it allows library dependencies to be unlocked much sooner, increasing parallelism during compilation. + For each rlib/lib, a separate RustcMetadata action produces a hollow rlib + (via `-Zno-codegen`) containing only metadata. Downstream rlib/lib crates + can begin compiling against the hollow rlib before the upstream full codegen + action completes, increasing build parallelism. + + Pipelining applies to rlib→rlib dependencies by default. To also pipeline + bin/cdylib crates (starting their compile step before upstream full codegen + finishes), enable `experimental_use_cc_common_link` alongside this flag. + With cc_common.link, rustc only emits `.o` files for binaries (linking is + handled separately), so hollow rlib deps are safe for bins too. """ bool_flag( name = "pipelined_compilation", @@ -126,6 +134,11 @@ def pipelined_compilation(): def experimental_use_cc_common_link(): """A flag to control whether to link rust_binary and rust_test targets using \ cc_common.link instead of rustc. + + When combined with `pipelined_compilation`, bin/cdylib crates also participate + in the hollow-rlib dependency chain: rustc only emits `.o` files (linking is + done by cc_common.link and does not check SVH), so bin compile steps can start + as soon as upstream hollow rlibs are ready rather than waiting for full codegen. """ bool_flag( name = "experimental_use_cc_common_link", diff --git a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl index 36a3de891b..0f638c3ee6 100644 --- a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl +++ b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl @@ -1,8 +1,8 @@ """Unittests for rust rules.""" load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") -load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro") -load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements", "assert_list_contains_adjacent_elements_not") +load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") +load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements_not") load(":wrap.bzl", "wrap") ENABLE_PIPELINING = { @@ -22,49 +22,77 @@ def _second_lib_test_impl(ctx): rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - # Both actions should use the same --emit= - assert_argv_contains(env, rlib_action, "--emit=dep-info,link,metadata") - assert_argv_contains(env, metadata_action, "--emit=dep-info,link,metadata") + # Hollow rlib approach: Rustc action uses --emit=dep-info,link (no metadata). + assert_argv_contains(env, rlib_action, "--emit=dep-info,link") - # The metadata action should have a .rmeta as output and the rlib action a .rlib + # Metadata action uses --emit=link=-hollow.rlib (hollow rlib, .rlib extension). + # The .rlib extension is required so rustc reads it as an rlib archive (extracting + # lib.rmeta with optimized MIR). Using .rmeta extension causes E0786, and using + # --emit=metadata produces raw .rmeta without optimized MIR (causes "missing + # optimized MIR" errors on Rust 1.85+). + metadata_emit_link = [arg for arg in metadata_action.argv if arg.startswith("--emit=link=") and arg.endswith("-hollow.rlib")] + asserts.true( + env, + len(metadata_emit_link) == 1, + "expected --emit=link=*-hollow.rlib for hollow rlib, got: " + str([arg for arg in metadata_action.argv if arg.startswith("--emit=")]), + ) + + # The rlib action produces a .rlib; the metadata action produces a -hollow.rlib. path = rlib_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rlib"), - "expected Rustc to output .rlib, got " + path, + path.endswith(".rlib") and not path.endswith("-hollow.rlib"), + "expected Rustc to output .rlib (not hollow), got " + path, ) path = metadata_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rmeta"), - "expected RustcMetadata to output .rmeta, got " + path, + path.endswith("-hollow.rlib"), + "expected RustcMetadata to output -hollow.rlib, got " + path, ) - # Only the action building metadata should contain --rustc-quit-on-rmeta + # Neither action should use --rustc-quit-on-rmeta (hollow rlib exits naturally). assert_list_contains_adjacent_elements_not(env, rlib_action.argv, ["--rustc-quit-on-rmeta", "true"]) - assert_list_contains_adjacent_elements(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) - - # Check that both actions refer to the metadata of :first, not the rlib - extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + assert_list_contains_adjacent_elements_not(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) + + # The metadata action should use -Zno-codegen for the hollow rlib approach. + assert_argv_contains(env, metadata_action, "-Zno-codegen") + + # The Rustc action should NOT use -Zno-codegen. + no_codegen_in_rlib = [arg for arg in rlib_action.argv if arg == "-Zno-codegen"] + asserts.true(env, len(no_codegen_in_rlib) == 0, "Rustc action should not have -Zno-codegen") + + # The metadata action references first's hollow rlib for --extern (pipelining: starts + # before first's full codegen finishes). The Rustc action uses the full rlib for + # --extern so the full rlib's embedded SVH matches the full rlib that downstream + # binaries (without cc_common.link) see in their -Ldependency path. If both actions + # used the hollow rlib, nondeterministic proc macros could produce different SVHs + # for the hollow vs full rlib, causing E0460 in downstream binary builds. + extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith("-hollow.rlib")] asserts.true( env, len(extern_metadata) == 1, - "did not find a --extern=first=*.rmeta but expected one", + "did not find --extern=first=*-hollow.rlib for metadata action, got: " + str([arg for arg in metadata_action.argv if arg.startswith("--extern=first=")]), ) - extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + extern_rlib_full = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and not arg.endswith("-hollow.rlib")] asserts.true( env, - len(extern_rlib) == 1, - "did not find a --extern=first=*.rlib but expected one", + len(extern_rlib_full) == 1, + "expected --extern=first=libfirst*.rlib (full rlib) for rlib action, got: " + str([arg for arg in rlib_action.argv if arg.startswith("--extern=first=")]), ) - # Check that the input to both actions is the metadata of :first + # The metadata action's input is first's hollow rlib only (no full rlib needed). input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input, found " + str([i.path for i in input_metadata])) - asserts.true(env, input_metadata[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_metadata[0].path) - input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_rlib) == 1, "expected only one libfirst input, found " + str([i.path for i in input_rlib])) - asserts.true(env, input_rlib[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_rlib[0].path) + asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input for metadata, found " + str([i.path for i in input_metadata])) + asserts.true(env, input_metadata[0].basename.endswith("-hollow.rlib"), "expected hollow rlib for metadata action, found " + input_metadata[0].path) + + # The Rustc action's inputs contain the full rlib (referenced by --extern) and the + # hollow rlib (present in the sandbox for -Ldependency=<_hollow_dir> resolution of + # transitive deps that were compiled against hollow rlibs). + input_rlib_full = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and not i.basename.endswith("-hollow.rlib")] + input_rlib_hollow = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and i.basename.endswith("-hollow.rlib")] + asserts.true(env, len(input_rlib_full) == 1, "expected full rlib in rlib action inputs, found " + str([i.path for i in input_rlib_full])) + asserts.true(env, len(input_rlib_hollow) == 1, "expected hollow rlib in rlib action inputs (for sandbox), found " + str([i.path for i in input_rlib_hollow])) return analysistest.end(env) @@ -124,10 +152,16 @@ def _pipelined_compilation_test(): target_under_test = ":bin", target_compatible_with = _NO_WINDOWS, ) + hollow_rlib_env_test( + name = "hollow_rlib_env_test", + target_under_test = ":second", + target_compatible_with = _NO_WINDOWS, + ) return [ ":second_lib_test", ":bin_test", + ":hollow_rlib_env_test", ] def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): @@ -138,8 +172,8 @@ def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): # also depend on metadata for 'wrapper'. rust_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rmeta")] - rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib")] + metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith("-hollow.rlib")] + rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib") and not i.path.endswith("-hollow.rlib")] seen_wrapper_metadata = False seen_to_wrap_metadata = False @@ -176,22 +210,30 @@ def _rmeta_is_used_when_building_custom_rule_test_impl(ctx): # This is the custom rule invocation of rustc. rust_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] - # We want to check that the action depends on metadata, regardless of ctx.attr.generate_metadata seen_to_wrap_rlib = False - seen_to_wrap_rmeta = False + seen_to_wrap_hollow = False for act in rust_action.inputs.to_list(): - if "libto_wrap" in act.path and act.path.endswith(".rlib"): + if "libto_wrap" in act.path and act.path.endswith("-hollow.rlib"): + seen_to_wrap_hollow = True + elif "libto_wrap" in act.path and act.path.endswith(".rlib") and not act.path.endswith("-hollow.rlib"): seen_to_wrap_rlib = True - elif "libto_wrap" in act.path and act.path.endswith(".rmeta"): - seen_to_wrap_rmeta = True - asserts.true(env, seen_to_wrap_rmeta, "expected dependency on metadata for 'to_wrap' but not found") - asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") + if ctx.attr.generate_metadata: + # When wrapper generates its own hollow rlib, the Rustc action uses the full + # rlib of to_wrap for --extern (SVH consistency) and also has the hollow rlib + # in the sandbox for -Ldependency= resolution. + asserts.true(env, seen_to_wrap_hollow, "expected hollow rlib in inputs (for sandbox) when generate_metadata=True") + asserts.true(env, seen_to_wrap_rlib, "expected full rlib in inputs for --extern when generate_metadata=True") + else: + # When wrapper does not generate its own hollow rlib, the Rustc action uses + # hollow rlib deps via normal _depend_on_metadata logic (pipelined rlib deps). + asserts.true(env, seen_to_wrap_hollow, "expected dependency on metadata for 'to_wrap' but not found") + asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") return analysistest.end(env) rmeta_is_propagated_through_custom_rule_test = analysistest.make(_rmeta_is_propagated_through_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) -rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, config_settings = ENABLE_PIPELINING) +rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): env = analysistest.begin(ctx) @@ -204,6 +246,33 @@ def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): rmeta_not_produced_if_pipelining_disabled_test = analysistest.make(_rmeta_not_produced_if_pipelining_disabled_test_impl, config_settings = ENABLE_PIPELINING) +def _hollow_rlib_env_test_impl(ctx): + """Verify RUSTC_BOOTSTRAP=1 is set consistently on both Rustc and RustcMetadata actions. + + RUSTC_BOOTSTRAP=1 changes the crate hash (SVH), so it must be set on both actions + to keep the hollow rlib and full rlib SVHs consistent.""" + env = analysistest.begin(ctx) + tut = analysistest.target_under_test(env) + metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] + rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] + + asserts.equals( + env, + "1", + metadata_action.env.get("RUSTC_BOOTSTRAP", ""), + "Metadata action should have RUSTC_BOOTSTRAP=1 for hollow rlib approach", + ) + asserts.equals( + env, + "1", + rlib_action.env.get("RUSTC_BOOTSTRAP", ""), + "Rustc action should have RUSTC_BOOTSTRAP=1 for SVH compatibility with hollow rlib", + ) + + return analysistest.end(env) + +hollow_rlib_env_test = analysistest.make(_hollow_rlib_env_test_impl, config_settings = ENABLE_PIPELINING) + def _disable_pipelining_test(): rust_library( name = "lib", @@ -249,6 +318,7 @@ def _custom_rule_test(generate_metadata, suffix): rmeta_is_used_when_building_custom_rule_test( name = "rmeta_is_used_when_building_custom_rule_test" + suffix, + generate_metadata = generate_metadata, target_under_test = ":wrapper" + suffix, target_compatible_with = _NO_WINDOWS, ) @@ -258,6 +328,59 @@ def _custom_rule_test(generate_metadata, suffix): ":rmeta_is_used_when_building_custom_rule_test" + suffix, ] +def _svh_mismatch_test(): + """Creates a rust_test demonstrating SVH mismatch with non-deterministic proc macros. + + Without pipelining (default): each library is compiled exactly once, SVH + is consistent across the dependency graph, and the test builds and passes. + + With pipelining (//rust/settings:pipelined_compilation=true): rules_rust + compiles svh_lib twice in separate rustc invocations — once for the hollow + metadata (.rmeta), once for the full .rlib. Because the proc macro uses + HashMap with OS-seeded randomness, these two invocations typically produce + different token streams and therefore different SVH values. The consumer is + compiled against the hollow .rmeta (recording SVH_1); when rustc links the + test binary against the full .rlib (SVH_2), it detects SVH_1 ≠ SVH_2 and + fails with E0460. The test is therefore expected to FAIL TO BUILD most of + the time (~99.2% with 5 HashMap entries) when pipelining is enabled. + + The test is marked flaky because the SVH mismatch is non-deterministic: + on rare occasions (~0.8%) both rustc invocations produce the same HashMap + iteration order and the build succeeds even with pipelining enabled. + """ + + rust_proc_macro( + name = "svh_nondeterministic_macro", + srcs = ["svh_mismatch/svh_mismatch_nondeterministic_macro.rs"], + crate_name = "nondeterministic_macro", + edition = "2021", + ) + + rust_library( + name = "svh_lib", + srcs = ["svh_mismatch/svh_mismatch_lib.rs"], + edition = "2021", + proc_macro_deps = [":svh_nondeterministic_macro"], + ) + + rust_library( + name = "svh_consumer", + srcs = ["svh_mismatch/svh_mismatch_consumer.rs"], + edition = "2021", + deps = [":svh_lib"], + ) + + rust_test( + name = "svh_mismatch_test", + srcs = ["svh_mismatch/svh_mismatch_test.rs"], + edition = "2021", + deps = [":svh_consumer"], + flaky = True, + target_compatible_with = _NO_WINDOWS, + ) + + return [":svh_mismatch_test"] + def pipelined_compilation_test_suite(name): """Entry-point macro called from the BUILD file. @@ -269,6 +392,7 @@ def pipelined_compilation_test_suite(name): tests.extend(_disable_pipelining_test()) tests.extend(_custom_rule_test(generate_metadata = True, suffix = "_with_metadata")) tests.extend(_custom_rule_test(generate_metadata = False, suffix = "_without_metadata")) + tests.extend(_svh_mismatch_test()) native.test_suite( name = name, diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs new file mode 100644 index 0000000000..99b0ea9bf4 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs @@ -0,0 +1,6 @@ +/// A library that depends on svh_lib. When compiled against a hollow `.rmeta` +/// of svh_lib, this crate's metadata records svh_lib's SVH at that point in +/// time. If the full `.rlib` of svh_lib was produced by a separate rustc +/// invocation (with a different HashMap seed), it may have a different SVH, +/// causing a mismatch when a downstream binary tries to link against both. +pub use svh_lib::Widget; diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs new file mode 100644 index 0000000000..e2f3985399 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs @@ -0,0 +1,8 @@ +use nondeterministic_macro::NondeterministicHash; + +/// A struct whose derivation runs the non-deterministic proc macro. +/// The macro generates a public constant whose value depends on HashMap +/// iteration order, so this crate's SVH varies between separate rustc +/// invocations. +#[derive(NondeterministicHash)] +pub struct Widget; diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs new file mode 100644 index 0000000000..7ba44425b7 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs @@ -0,0 +1,37 @@ +extern crate proc_macro; +use proc_macro::TokenStream; +use std::collections::HashMap; + +/// A derive macro that produces non-deterministic output due to HashMap's +/// random iteration order. Each separate process invocation initializes +/// `HashMap` with a different OS-seeded `RandomState`, so iteration order +/// varies between invocations. This makes the generated constant—and thus +/// the crate's SVH—differ when the macro is run twice (e.g., once for a +/// hollow `.rmeta` and once for a full `.rlib` in pipelined compilation). +#[proc_macro_derive(NondeterministicHash)] +pub fn nondeterministic_hash_derive(_input: TokenStream) -> TokenStream { + // HashMap::new() uses RandomState, which seeds from OS entropy. + // Each separate process invocation gets a different seed, so iteration + // order over the map is non-deterministic across invocations. + let mut map = HashMap::new(); + map.insert("alpha", 1u64); + map.insert("beta", 2u64); + map.insert("gamma", 4u64); + map.insert("delta", 8u64); + map.insert("epsilon", 16u64); + + // Position-weighted sum: not commutative, so different iteration orders + // produce different values. With 5 entries (5! = 120 orderings), the + // probability of identical output in two separate invocations is ~0.8%. + let fingerprint: u64 = map + .iter() + .enumerate() + .map(|(pos, (_, &val))| val.wrapping_mul(pos as u64 + 1)) + .fold(0u64, u64::wrapping_add); + + // Exposing this as a public constant makes it part of the crate's + // exported API, which is included in the SVH computation. + format!("pub const NONDETERMINISTIC_HASH_FINGERPRINT: u64 = {};", fingerprint) + .parse() + .unwrap() +} diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs new file mode 100644 index 0000000000..6ecfe83553 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs @@ -0,0 +1,28 @@ +/// Demonstrates SVH (Strict Version Hash) mismatch with pipelined compilation. +/// +/// Without pipelining this test always builds and passes: each library is +/// compiled exactly once, so the SVH embedded in every `.rmeta` and `.rlib` +/// is identical. +/// +/// With `//rust/settings:pipelined_compilation=true` rules_rust compiles +/// `svh_lib` **twice** in separate rustc processes — once to emit the hollow +/// `.rmeta` (metadata only), once to emit the full `.rlib`. Because +/// `nondeterministic_macro` uses `HashMap` with OS-seeded randomness, the two +/// rustc invocations typically produce different token streams and therefore +/// different SVH values. `svh_consumer` is compiled against the hollow `.rmeta` +/// and records SVH_1 in its own metadata; when rustc later tries to link the +/// test binary against the full `.rlib` (which carries SVH_2), it detects the +/// mismatch and fails with E0460. The test therefore **fails to build** most of +/// the time (~99.2% probability) when pipelining is enabled. +/// +/// The `flaky = True` attribute on this target acknowledges that the mismatch +/// is non-deterministic: on rare occasions (~0.8%) both rustc invocations +/// happen to produce the same HashMap iteration order, the SVHs agree, and the +/// build succeeds. +use svh_consumer::Widget; + +#[test] +fn svh_consistent() { + // If we reach here the SVH was consistent (no pipelining, or a lucky run). + let _: Widget = Widget; +} diff --git a/test/unit/pipelined_compilation/wrap.bzl b/test/unit/pipelined_compilation/wrap.bzl index f24a0e421a..e3f4ac5482 100644 --- a/test/unit/pipelined_compilation/wrap.bzl +++ b/test/unit/pipelined_compilation/wrap.bzl @@ -40,12 +40,23 @@ def _wrap_impl(ctx): lib_hash = output_hash, extension = ".rlib", ) - rust_metadata_name = "{prefix}{name}-{lib_hash}{extension}".format( - prefix = "lib", - name = crate_name, - lib_hash = output_hash, - extension = ".rmeta", - ) + + # Use -hollow.rlib extension (not .rmeta) so rustc reads it as an rlib archive + # containing optimized MIR. See rust/private/rust.bzl for the same logic. + # The hollow rlib is placed in a "_hollow/" subdirectory to avoid the full rlib + # and hollow rlib appearing in the same -Ldependency= search directory, which + # would cause E0463 "can't find crate" errors due to ambiguous crate candidates. + metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, crate_type) and ctx.attr.generate_metadata + if metadata_supports_pipelining: + rust_metadata_name = "_hollow/lib{name}-{lib_hash}-hollow.rlib".format( + name = crate_name, + lib_hash = output_hash, + ) + else: + rust_metadata_name = "lib{name}-{lib_hash}.rmeta".format( + name = crate_name, + lib_hash = output_hash, + ) tgt = ctx.attr.target deps = [DepVariantInfo( @@ -73,8 +84,7 @@ def _wrap_impl(ctx): aliases = {}, output = rust_lib, metadata = rust_metadata, - metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, crate_type) and - ctx.attr.generate_metadata, + metadata_supports_pipelining = metadata_supports_pipelining, owner = ctx.label, edition = "2018", compile_data = depset([]),