Stdlib source bootstrap

Author: dzbarsky

MODULE.bazel

@@ -57,7 +57,7 @@ bazel_dep(name = "platforms", version = "1.1.0")
 bazel_dep(name = "protobuf", version = "34.0.bcr.1", repo_name = "com_google_protobuf")
 bazel_dep(name = "rules_cc", version = "0.2.8")
 bazel_dep(name = "rules_proto", version = "7.1.0")
-bazel_dep(name = "llvm", version = "0.7.7")
+bazel_dep(name = "llvm", version = "0.7.8")
 bazel_dep(name = "aspect_tools_telemetry", version = "0.3.3")
 
 osx = use_extension("@llvm//extensions:osx.bzl", "osx")

MODULE.bazel.lock

@@ -1,8 +1,8 @@
 """Config settings for Rust target triples."""
 
-load("//rs/platforms:triples.bzl", "SUPPORTED_TARGET_TRIPLES", "triple_to_constraint_set")
+load("//rs/platforms:triples.bzl", "ALL_TARGET_TRIPLES", "triple_to_constraint_set")
 
-def declare_config_settings(targets = SUPPORTED_TARGET_TRIPLES):
+def declare_config_settings(targets = ALL_TARGET_TRIPLES):
     for target_triple in targets:
         native.config_setting(
             name = target_triple,

rs/platforms/config/declare_config_settings.bzl

@@ -1,8 +1,8 @@
 """Platform definitions for Rust target triples."""
 
-load("//rs/platforms:triples.bzl", "SUPPORTED_TARGET_TRIPLES", "triple_to_constraint_set")
+load("//rs/platforms:triples.bzl", "ALL_TARGET_TRIPLES", "triple_to_constraint_set")
 
-def declare_platforms(targets = SUPPORTED_TARGET_TRIPLES):
+def declare_platforms(targets = ALL_TARGET_TRIPLES):
     for target_triple in targets:
         native.platform(
             name = target_triple,

rs/platforms/declare_platforms.bzl

@@ -9,6 +9,10 @@ def triple_to_constraint_set(target_triple):
     t = triple(target_triple)
 
     if t.system in ("linux", "nixos"):
+        # TODO: Add LLVM constraint support for libc-less Linux builds.
+        # if t.abi == "none":
+        #     constraints.append("@llvm//constraints/libc:unconstrained")
+        # elif t.abi == "musl" or "musl" in target_triple:
         if t.abi == "musl" or "musl" in target_triple:
             # Rustc passes `-no-pie` on musl so make sure we align.
             constraints.append("@llvm//constraints/libc:musl")
@@ -143,5 +147,43 @@ SUPPORTED_TARGET_TRIPLES = [
     #"x86_64-unknown-redox",             # ✓ Redox OS
     "x86_64-unknown-uefi",  # ? 64-bit UEFI
 
-    # No Tier3 support.
+    # Additional Tier 3 targets must opt into source stdlib builds.
 ]
+
+SOURCE_STDLIB_TARGET_TRIPLES = [
+    "aarch64-unknown-freebsd",
+    "aarch64-unknown-netbsd",
+    "aarch64-unknown-nto-qnx710",
+    "aarch64-unknown-openbsd",
+    "arm64e-apple-darwin",
+    "arm64e-apple-ios",
+    "armv7-unknown-freebsd",
+    "armv7-unknown-netbsd-eabihf",
+    "bpfeb-unknown-none",
+    "bpfel-unknown-none",
+    "i386-apple-ios",
+    "i686-apple-darwin",
+    "i686-unknown-netbsd",
+    "i686-unknown-openbsd",
+    "powerpc-unknown-freebsd",
+    "powerpc-unknown-linux-musl",
+    "powerpc-unknown-netbsd",
+    "powerpc-unknown-openbsd",
+    "powerpc64-unknown-freebsd",
+    "powerpc64-unknown-openbsd",
+    "powerpc64le-unknown-freebsd",
+    "riscv64-linux-android",
+    "riscv64gc-unknown-freebsd",
+    "riscv64gc-unknown-fuchsia",
+    "riscv64gc-unknown-netbsd",
+    "riscv64gc-unknown-openbsd",
+    "s390x-unknown-linux-musl",
+    "sparc64-unknown-netbsd",
+    "sparc64-unknown-openbsd",
+    "wasm64-unknown-unknown",
+    # TODO: Enable once @llvm has constraint support for libc-less Linux builds.
+    # "x86_64-unknown-linux-none",
+    "x86_64-unknown-openbsd",
+]
+
+ALL_TARGET_TRIPLES = SUPPORTED_TARGET_TRIPLES + SOURCE_STDLIB_TARGET_TRIPLES

rs/platforms/triples.bzl

@@ -1,9 +1,31 @@
 load("@bazel_lib//:bzl_library.bzl", "bzl_library")
+load("@bazel_skylib//rules:common_settings.bzl", "bool_setting")
+load("@rules_rust//rust:defs.bzl", "rust_stdlib_filegroup")
 load(":all_crate_deps_test.bzl", "all_crate_deps_tests")
 load(":cfg_parser_test.bzl", "cfg_parser_tests")
 load(":lint_flags_test.bzl", "lint_flags_tests")
 load(":semver_test.bzl", "semver_select_tests")
 
+bool_setting(
+    name = "source_stdlib_building",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "is_source_stdlib_building",
+    flag_values = {
+        ":source_stdlib_building": "true",
+    },
+    visibility = ["//visibility:public"],
+)
+
+rust_stdlib_filegroup(
+    name = "empty_stdlib",
+    srcs = [],
+    visibility = ["//visibility:public"],
+)
+
 all_crate_deps_tests()
 
 cfg_parser_tests()
@@ -109,6 +131,13 @@ bzl_library(
     deps = ["@bazel_lib//lib:repo_utils"],
 )
 
+bzl_library(
+    name = "source_stdlib",
+    srcs = ["source_stdlib.bzl"],
+    visibility = ["//rs:__subpackages__"],
+    deps = ["@rules_rust//rust:bzl_lib"],
+)
+
 bzl_library(
     name = "git_cargo_workspace_repository",
     srcs = ["git_cargo_workspace_repository.bzl"],
@@ -126,6 +155,7 @@ bzl_library(
     srcs = ["cargo_repository.bzl"],
     visibility = ["//rs:__subpackages__"],
     deps = [
+        "//rs/private:rust_repository_utils",
         "@rules_rust//rust/platform:bzl_lib",
         "@rules_rust//rust/private:bzl_lib",
     ],
@@ -168,6 +198,11 @@ bzl_library(
     srcs = ["rust_src_repository.bzl"],
     visibility = ["//rs:__subpackages__"],
     deps = [
+        ":cargo_workspace_graph",
+        ":repository_utils",
+        ":toml2json",
+        "//rs/platforms:triples",
+        "@bazel_skylib//lib:paths",
         "@bazel_tools//tools/build_defs/repo:utils.bzl",
         "@rules_rust//rust/private:bzl_lib",
     ],
@@ -216,16 +251,6 @@ bzl_library(
     ],
 )
 
-bzl_library(
-    name = "symlink_utils",
-    srcs = ["symlink_utils.bzl"],
-    visibility = ["//rs:__subpackages__"],
-    deps = [
-        "@bazel_features//:features",
-        "@bazel_lib//lib:paths",
-    ],
-)
-
 bzl_library(
     name = "cargo_lints",
     srcs = ["cargo_lints.bzl"],

rs/private/BUILD.bazel

@@ -239,6 +239,7 @@ def resolve_cargo_workspace_members(
         debug,
         dep_label_prefix = None,
         allow_missing_resolved_deps = False,
+        skip_internal_rustc_placeholder_crates = True,
         watch_manifests = False,
         use_legacy_rules_rust_platforms = False):
     platform_cfg_attrs = [triple_to_cfg_attrs(triple) for triple in platform_triples]
@@ -248,9 +249,10 @@ def resolve_cargo_workspace_members(
 
     cfg_match_cache = {None: struct(matches = platform_triples, uses_feature_cfg = False)}
 
-    workspace_member_keys = {}
-    for package in cargo_metadata["packages"]:
-        workspace_member_keys[(package["name"], package["version"])] = True
+    workspace_member_keys = set([
+        (package["name"], package["version"])
+        for package in cargo_metadata["packages"]
+    ])
 
     resolver_versions_by_name = {name: versions[:] for name, versions in versions_by_name.items()}
     workspace_members_by_key = {(package["name"], package["version"]): package for package in workspace_members}
@@ -270,6 +272,7 @@ def resolve_cargo_workspace_members(
         possible_deps = prepare_possible_deps(
             package.get("dependencies", []),
             converter = cargo_metadata_dep_to_dep_dict,
+            skip_internal_rustc_placeholder_crates = skip_internal_rustc_placeholder_crates,
         )
 
         package_index = len(resolver_packages)

rs/private/cargo_workspace_graph.bzl

@@ -1,34 +1,8 @@
 load("@bazel_tools//tools/build_defs/repo:git_worker.bzl", "git_repo")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "patch")
-load(":repository_utils.bzl", "cargo_build_file_values")
+load(":repository_utils.bzl", "cargo_build_file_values", "inherit_workspace_package_fields")
 load(":toml2json.bzl", "run_toml2json")
 
-_INHERITABLE_FIELDS = [
-    "version",
-    "edition",
-    "description",
-    "homepage",
-    "repository",
-    "license",
-    # TODO(zbarsky): Do we need to fixup the path for readme and license_file?
-    "license_file",
-    "rust_version",
-    "readme",
-]
-
-def _inherit_workspace_package_fields(cargo_toml, workspace_cargo_toml):
-    workspace_package = workspace_cargo_toml.get("workspace", {}).get("package")
-    if not workspace_package:
-        return cargo_toml
-
-    crate_package = cargo_toml["package"]
-    for field in _INHERITABLE_FIELDS:
-        value = crate_package.get(field)
-        if type(value) == "dict" and value.get("workspace") == True:
-            crate_package[field] = workspace_package.get(field)
-
-    return cargo_toml
-
 def _render_label_list(labels):
     return ",\n        ".join(['"%s"' % label for label in sorted(labels)])
 
@@ -42,7 +16,7 @@ def _render_build_file(rctx, dest, additive_build_file_content, gen_binaries, wo
     package_path = rctx.path(dest).dirname
     cargo_toml_path = package_path.get_child("Cargo.toml")
     cargo_toml = run_toml2json(rctx, cargo_toml_path)
-    cargo_toml = _inherit_workspace_package_fields(cargo_toml, workspace_cargo_toml)
+    cargo_toml = inherit_workspace_package_fields(cargo_toml, workspace_cargo_toml)
     package = cargo_toml["package"]
 
     cargo = cargo_build_file_values(

rs/private/git_cargo_workspace_repository.bzl

@@ -1,20 +1,12 @@
 def _host_tools_repository_impl(rctx):
-    defs_bzl_content = """\
-RS_HOST_CARGO_LABEL = Label("@{host_cargo_repo}//:bin/cargo{binary_suffix}")
-""".format(
-        host_cargo_repo = rctx.attr.host_cargo_repo,
-        binary_suffix = rctx.attr.binary_suffix,
-    )
-
-    rctx.file("defs.bzl", defs_bzl_content)
+    rctx.file("defs.bzl", 'RS_HOST_CARGO_LABEL = Label("%s")\n' % rctx.attr.host_cargo)
     rctx.file("BUILD.bazel", 'exports_files(["defs.bzl"])')
 
     return rctx.repo_metadata(reproducible = True)
 
 host_tools_repository = repository_rule(
     implementation = _host_tools_repository_impl,
     attrs = {
-        "host_cargo_repo": attr.string(mandatory = True),
-        "binary_suffix": attr.string(mandatory = True),
+        "host_cargo": attr.label(allow_single_file = True, mandatory = True),
     },
 )

rs/private/host_tools_repository.bzl

@@ -40,6 +40,32 @@ def render_select_build_script_env(platform_items, use_legacy_rules_rust_platfor
 def _exclude_deps_from_features(features):
     return [f for f in features if not f.startswith("dep:")]
 
+_INHERITABLE_PACKAGE_FIELDS = [
+    "version",
+    "edition",
+    "description",
+    "homepage",
+    "repository",
+    "license",
+    # TODO(zbarsky): Do we need to fixup the path for readme and license_file?
+    "license_file",
+    "rust_version",
+    "readme",
+]
+
+def inherit_workspace_package_fields(cargo_toml, workspace_cargo_toml):
+    workspace_package = workspace_cargo_toml.get("workspace", {}).get("package")
+    if not workspace_package:
+        return cargo_toml
+
+    crate_package = cargo_toml["package"]
+    for field in _INHERITABLE_PACKAGE_FIELDS:
+        value = crate_package.get(field)
+        if type(value) == "dict" and value.get("workspace") == True:
+            crate_package[field] = workspace_package.get(field)
+
+    return cargo_toml
+
 def cargo_build_file_values(rctx, cargo_toml, gen_binaries, package_path = "", gen_build_script = None):
     package_dir = rctx.path(package_path or ".")
     package = cargo_toml["package"]
@@ -141,11 +167,15 @@ _RUST_CRATE_MACRO_CALL = """{indent}rust_crate(
 {indent}    data = [
 {indent}        {data}
 {indent}    ],
+{indent}    extra_compile_data = [
+{indent}        {extra_compile_data}
+{indent}    ],
 {indent}    crate_features = {crate_features},
 {indent}    triples = {triples},
 {indent}    conditional_crate_features = {conditional_crate_features},
 {indent}    crate_root = {crate_root},
 {indent}    edition = {edition},
+{indent}    rustc_env = {rustc_env},
 {indent}    rustc_flags = {rustc_flags}{conditional_rustc_flags},
 {indent}    tags = {tags},
 {indent}    target_compatible_with = RESOLVED_PLATFORMS,
@@ -162,10 +192,12 @@ _RUST_CRATE_MACRO_CALL = """{indent}rust_crate(
 {indent}    is_proc_macro = {is_proc_macro},
 {indent}    binaries = {binaries},
 {indent}    use_legacy_rules_rust_platforms = {use_legacy_rules_rust_platforms},
+{indent}    cargo_toml_env = {cargo_toml_env},
+{indent}    skip_deps_verification = {skip_deps_verification},
 {indent})
 """
 
-def render_rust_crate_call(attr, values, bazel_metadata = {}, extra_deps = "", indent = ""):
+def render_rust_crate_call(attr, values, bazel_metadata = {}, extra_deps = "", indent = "", skip_deps_verification = False):
     # We keep conditional_crate_features unrendered here because it must be treated specially for build scripts.
     # See `rust_crate.bzl` for details.
     crate_features, conditional_crate_features = compute_select(
@@ -195,11 +227,13 @@ def render_rust_crate_call(attr, values, bazel_metadata = {}, extra_deps = "", i
         extra_deps = extra_deps,
         conditional_deps = " + " + conditional_deps if conditional_deps else "",
         data = list_indent.join(['"%s"' % d for d in attr.data]),
+        extra_compile_data = list_indent.join(['"%s"' % d for d in attr.extra_compile_data]),
         crate_features = repr(sorted(crate_features)),
         triples = repr(attr.crate_features_select.keys()),
         conditional_crate_features = repr(conditional_crate_features),
         crate_root = values["crate_root"],
         edition = values["edition"],
+        rustc_env = repr(attr.rustc_env),
         rustc_flags = repr(rustc_flags),
         conditional_rustc_flags = " + " + conditional_rustc_flags if conditional_rustc_flags else "",
         tags = repr(attr.crate_tags),
@@ -218,6 +252,8 @@ def render_rust_crate_call(attr, values, bazel_metadata = {}, extra_deps = "", i
         is_proc_macro = values["is_proc_macro"],
         binaries = values["binaries"],
         use_legacy_rules_rust_platforms = use_legacy_rules_rust_platforms,
+        cargo_toml_env = attr.cargo_toml_env,
+        skip_deps_verification = repr(skip_deps_verification),
     )
 
 def render_build_file_content(rctx, attr, values, bazel_metadata = {}):
@@ -250,16 +286,19 @@ rust_crate_attrs = {
     "build_script_tools": attr.label_list(default = []),
     "build_script_tools_select": attr.string_list_dict(),
     "build_script_tags": attr.string_list(),
+    "rustc_env": attr.string_dict(),
     "rustc_flags": attr.string_list(),
     "rustc_flags_select": attr.string_list_dict(),
     "crate_tags": attr.string_list(),
     "data": attr.label_list(default = []),
+    "extra_compile_data": attr.label_list(default = []),
     "deps": attr.string_list(default = []),
     "deps_select": attr.string_list_dict(),
     "aliases": attr.string_dict(),
     "crate_features": attr.string_list(),
     "crate_features_select": attr.string_list_dict(),
     "use_legacy_rules_rust_platforms": attr.bool(),
+    "cargo_toml_env": attr.bool(default = True),
 }
 
 common_attrs = rust_crate_attrs | {