Bootstrap source stdlib from rust-src

Author: dzbarsky

MODULE.bazel

@@ -58,7 +58,7 @@ bazel_dep(name = "protobuf", version = "34.0.bcr.1", repo_name = "com_google_pro
 bazel_dep(name = "rules_cc", version = "0.2.8")
 bazel_dep(name = "rules_proto", version = "7.1.0")
 bazel_dep(name = "rules_python", version = "1.6.3")
-bazel_dep(name = "llvm", version = "0.7.7")
+bazel_dep(name = "llvm", version = "0.8.0")
 bazel_dep(name = "aspect_tools_telemetry", version = "0.3.3")
 
 osx = use_extension("@llvm//extensions:osx.bzl", "osx")

MODULE.bazel.lock

@@ -1,8 +1,8 @@
 """Config settings for Rust target triples."""
 
-load("//rs/platforms:triples.bzl", "SUPPORTED_TARGET_TRIPLES", "triple_to_constraint_set")
+load("//rs/platforms:triples.bzl", "ALL_TARGET_TRIPLES", "triple_to_constraint_set")
 
-def declare_config_settings(targets = SUPPORTED_TARGET_TRIPLES):
+def declare_config_settings(targets = ALL_TARGET_TRIPLES):
     for target_triple in targets:
         native.config_setting(
             name = target_triple,

rs/platforms/config/declare_config_settings.bzl

@@ -1,8 +1,8 @@
 """Platform definitions for Rust target triples."""
 
-load("//rs/platforms:triples.bzl", "SUPPORTED_TARGET_TRIPLES", "triple_to_constraint_set")
+load("//rs/platforms:triples.bzl", "ALL_TARGET_TRIPLES", "triple_to_constraint_set")
 
-def declare_platforms(targets = SUPPORTED_TARGET_TRIPLES):
+def declare_platforms(targets = ALL_TARGET_TRIPLES):
     for target_triple in targets:
         native.platform(
             name = target_triple,

rs/platforms/declare_platforms.bzl

@@ -36,7 +36,7 @@ SUPPORTED_EXEC_TRIPLES = [
 ]
 
 # See https://doc.rust-lang.org/beta/rustc/platform-support.html
-SUPPORTED_TARGET_TRIPLES = [
+SUPPORTED_TIER_1_AND_2_TRIPLES = [
     # Tier 1
     "aarch64-apple-darwin",  # ARM64 macOS (11.0+, Big Sur+)
     "aarch64-pc-windows-msvc",  # ARM64 Windows MSVC
@@ -142,6 +142,40 @@ SUPPORTED_TARGET_TRIPLES = [
     "x86_64-unknown-none",  # * Freestanding/bare-metal x86_64, softfloat
     #"x86_64-unknown-redox",             # ✓ Redox OS
     "x86_64-unknown-uefi",  # ? 64-bit UEFI
+]
 
-    # No Tier3 support.
+SUPPORTED_TIER_3_TRIPLES = [
+    "aarch64-unknown-freebsd",
+    "aarch64-unknown-netbsd",
+    "aarch64-unknown-nto-qnx710",
+    "aarch64-unknown-openbsd",
+    "arm64e-apple-darwin",
+    "arm64e-apple-ios",
+    "armv7-unknown-freebsd",
+    "armv7-unknown-netbsd-eabihf",
+    "bpfeb-unknown-none",
+    "bpfel-unknown-none",
+    "i386-apple-ios",
+    "i686-apple-darwin",
+    "i686-unknown-netbsd",
+    "i686-unknown-openbsd",
+    "powerpc-unknown-freebsd",
+    "powerpc-unknown-linux-musl",
+    "powerpc-unknown-netbsd",
+    "powerpc-unknown-openbsd",
+    "powerpc64-unknown-freebsd",
+    "powerpc64-unknown-openbsd",
+    "powerpc64le-unknown-freebsd",
+    "riscv64-linux-android",
+    "riscv64gc-unknown-freebsd",
+    "riscv64gc-unknown-fuchsia",
+    "riscv64gc-unknown-netbsd",
+    "riscv64gc-unknown-openbsd",
+    "s390x-unknown-linux-musl",
+    "sparc64-unknown-netbsd",
+    "sparc64-unknown-openbsd",
+    "wasm64-unknown-unknown",
+    "x86_64-unknown-openbsd",
 ]
+
+ALL_TARGET_TRIPLES = SUPPORTED_TIER_1_AND_2_TRIPLES + SUPPORTED_TIER_3_TRIPLES

rs/platforms/triples.bzl

@@ -1,11 +1,36 @@
 load("@bazel_lib//:bzl_library.bzl", "bzl_library")
+load("@bazel_skylib//rules:common_settings.bzl", "bool_setting")
+load("@rules_rust//rust:defs.bzl", "rust_stdlib_filegroup")
 load(":all_crate_deps_test.bzl", "all_crate_deps_tests")
+load(":cargo_workspace_graph_test.bzl", "cargo_workspace_graph_tests")
 load(":cfg_parser_test.bzl", "cfg_parser_tests")
 load(":lint_flags_test.bzl", "lint_flags_tests")
 load(":semver_test.bzl", "semver_select_tests")
 
+bool_setting(
+    name = "source_stdlib_building",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "is_source_stdlib_building",
+    flag_values = {
+        ":source_stdlib_building": "true",
+    },
+    visibility = ["//visibility:public"],
+)
+
+rust_stdlib_filegroup(
+    name = "empty_stdlib",
+    srcs = [],
+    visibility = ["//visibility:public"],
+)
+
 all_crate_deps_tests()
 
+cargo_workspace_graph_tests()
+
 cfg_parser_tests()
 
 lint_flags_tests()
@@ -109,6 +134,13 @@ bzl_library(
     deps = ["@bazel_lib//lib:repo_utils"],
 )
 
+bzl_library(
+    name = "source_stdlib",
+    srcs = ["source_stdlib.bzl"],
+    visibility = ["//rs:__subpackages__"],
+    deps = ["@rules_rust//rust:bzl_lib"],
+)
+
 bzl_library(
     name = "git_cargo_workspace_repository",
     srcs = ["git_cargo_workspace_repository.bzl"],
@@ -169,6 +201,11 @@ bzl_library(
     srcs = ["rust_src_repository.bzl"],
     visibility = ["//rs:__subpackages__"],
     deps = [
+        ":cargo_workspace_graph",
+        ":repository_utils",
+        ":toml2json",
+        "//rs/platforms:triples",
+        "@bazel_skylib//lib:paths",
         "@bazel_tools//tools/build_defs/repo:utils.bzl",
         "@rules_rust//rust/private:bzl_lib",
     ],

rs/private/BUILD.bazel

@@ -110,7 +110,10 @@ def cargo_metadata_dep_to_dep_dict(dep):
 
 def _cargo_toml_dep_to_dep_dict_inner(dep, spec, is_build = False, target = None):
     if type(spec) == "string":
-        converted = {"name": dep}
+        converted = {
+            "name": dep,
+            "req": spec,
+        }
     else:
         converted = {
             "name": dep,
@@ -120,6 +123,8 @@ def _cargo_toml_dep_to_dep_dict_inner(dep, spec, is_build = False, target = None
         }
         if "package" in spec:
             converted["package"] = spec["package"]
+        if spec.get("version"):
+            converted["req"] = spec["version"]
 
     if is_build:
         converted["kind"] = "build"
@@ -205,8 +210,11 @@ def prepare_possible_deps(dependencies, converter = None, skip_internal_rustc_pl
 
     return possible_deps
 
+def dep_package_name(dep):
+    return dep.get("package") or dep["name"]
+
 def compute_package_fq_deps(package, versions_by_name, strict = True):
-    possible_dep_fq_crate_by_name = {}
+    possible_dep_fq_crates_by_name = {}
 
     for maybe_fq_dep in package.get("dependencies", []):
         idx = maybe_fq_dep.find(" ")
@@ -222,9 +230,36 @@ def compute_package_fq_deps(package, versions_by_name, strict = True):
             dep = maybe_fq_dep[:idx]
             resolved_version = maybe_fq_dep[idx + 1:]
 
-        possible_dep_fq_crate_by_name[dep] = fq_crate(dep, resolved_version)
+        existing = possible_dep_fq_crates_by_name.get(dep)
+        if not existing:
+            existing = []
+            possible_dep_fq_crates_by_name[dep] = existing
+        existing.append(fq_crate(dep, resolved_version))
+
+    return possible_dep_fq_crates_by_name
+
+def select_package_fq_dep(dep, fq_deps):
+    dep_package = dep_package_name(dep)
+    candidates = fq_deps.get(dep_package)
+    if not candidates:
+        return None
+
+    if len(candidates) == 1:
+        return candidates[0]
 
-    return possible_dep_fq_crate_by_name
+    req = dep.get("req")
+    if not req:
+        return None
+
+    versions = [
+        candidate[len(dep_package) + 1:]
+        for candidate in candidates
+    ]
+    version = select_matching_version(req, versions)
+    if not version:
+        return None
+
+    return fq_crate(dep_package, version)
 
 def compute_workspace_fq_deps(workspace_members, versions_by_name):
     workspace_fq_deps = {}
@@ -255,41 +290,55 @@ def _relative_to_workspace(path, workspace_root):
     rel_parts = [".."] * (len(root_parts) - common) + path_parts[common:]
     return "/".join(rel_parts) if rel_parts else "."
 
-def split_lockfile_packages(hub_name, cargo_metadata, workspace_cargo_toml_json, all_packages):
-    workspace_root = normalize_path(cargo_metadata["workspace_root"])
-    workspace_root_prefix = workspace_root + "/"
-
-    workspace_member_keys = {}
-    for package in cargo_metadata["packages"]:
-        workspace_member_keys[(package["name"], package["version"])] = True
+def cargo_metadata_dep_paths_by_name(cargo_metadata, workspace_root):
+    package_dirs = {}
 
-    dep_paths_by_name = {}
     for package in cargo_metadata["packages"]:
         for dep in package.get("dependencies", []):
             dep_path = dep.get("path")
-            if dep_path:
-                dep_paths_by_name[dep["name"]] = _relative_to_workspace(dep_path, workspace_root)
+            if not dep_path:
+                continue
+
+            package_dirs[dep["name"]] = _relative_to_workspace(dep_path, workspace_root)
+
+    return package_dirs
+
+def cargo_toml_patch_paths_by_name(workspace_cargo_toml, workspace_root, workspace_package_dir = ""):
+    workspace_root = normalize_path(workspace_root)
+    workspace_root_prefix = workspace_root + "/"
+    package_dirs = {}
 
-    patch_paths_by_name = {}
-    for registry_patches in workspace_cargo_toml_json.get("patch", {}).values():
-        for name, spec in registry_patches.items():
+    for patches in workspace_cargo_toml.get("patch", {}).values():
+        for name, spec in patches.items():
             if type(spec) != "dict":
                 continue
 
             patch_path = spec.get("path")
             if not patch_path:
                 continue
 
-            if patch_path.startswith("/"):
+            package = spec.get("package") or name
+            if paths.is_absolute(patch_path):
                 normalized = normalize_path(patch_path)
                 if not normalized.startswith(workspace_root_prefix):
                     fail("Patch path for %s points outside the workspace: %s" % (name, patch_path))
-                rel_patch_path = normalized.removeprefix(workspace_root_prefix)
+                package_dirs[package] = normalized.removeprefix(workspace_root_prefix)
             else:
-                rel_patch_path = normalize_path(paths.normalize(patch_path))
+                package_dirs[package] = normalize_path(paths.normalize(paths.join(workspace_package_dir, patch_path)))
 
-            patch_paths_by_name[name] = rel_patch_path
+    return package_dirs
 
+def split_lockfile_packages(hub_name, cargo_metadata, workspace_cargo_toml, all_packages, repo_root = None, workspace_package_dir = ""):
+    if repo_root == None:
+        repo_root = cargo_metadata["workspace_root"]
+    repo_root = normalize_path(repo_root)
+
+    workspace_member_keys = {}
+    for package in cargo_metadata["packages"]:
+        workspace_member_keys[(package["name"], package["version"])] = True
+
+    dep_paths_by_name = cargo_metadata_dep_paths_by_name(cargo_metadata, repo_root)
+    patch_paths_by_name = cargo_toml_patch_paths_by_name(workspace_cargo_toml, repo_root, workspace_package_dir)
     workspace_members = []
     packages = []
 
@@ -308,21 +357,22 @@ def split_lockfile_packages(hub_name, cargo_metadata, workspace_cargo_toml_json,
         rel_path = patch_paths_by_name.get(pkg["name"]) or dep_paths_by_name.get(pkg["name"])
         local_path = rel_path
         if rel_path and not rel_path.startswith("/"):
-            local_path = paths.join(workspace_root, rel_path)
+            local_path = paths.join(repo_root, rel_path)
 
         if not local_path:
             fail("Found a path dependency on %s %s but could not determine its path from Cargo.toml. Please declare it in [patch] or as a path dependency." % (pkg["name"], pkg["version"]))
 
         pkg["source"] = "path+" + hub_name + "/" + rel_path
         pkg["local_path"] = local_path
+        pkg["package_dir"] = rel_path
         packages.append(pkg)
 
     return struct(
         packages = packages,
         workspace_members = workspace_members,
     )
 
-def resolve_package_facts(packages, facts_by_fq_crate, platform_triples):
+def resolve_package_facts(packages, facts_by_fq_crate, platform_triples, skip_internal_rustc_placeholder_crates = True):
     feature_resolutions_by_fq_crate = {}
     versions_by_name = {}
 
@@ -335,7 +385,10 @@ def resolve_package_facts(packages, facts_by_fq_crate, platform_triples):
 
         fact = facts_by_fq_crate[fq_crate(name, version)]
         possible_features = fact["features"]
-        possible_deps = prepare_possible_deps(fact["dependencies"])
+        possible_deps = prepare_possible_deps(
+            fact["dependencies"],
+            skip_internal_rustc_placeholder_crates = skip_internal_rustc_placeholder_crates,
+        )
         feature_resolutions = new_feature_resolutions(package_index, possible_deps, possible_features, platform_triples)
         package["feature_resolutions"] = feature_resolutions
         feature_resolutions_by_fq_crate[fq_crate(name, version)] = feature_resolutions
@@ -423,6 +476,7 @@ def resolve_cargo_workspace_members(
         debug,
         dep_label_prefix = None,
         allow_missing_resolved_deps = False,
+        skip_internal_rustc_placeholder_crates = True,
         watch_manifests = False,
         use_legacy_rules_rust_platforms = False):
     platform_cfg_attrs = [triple_to_cfg_attrs(triple) for triple in platform_triples]
@@ -454,6 +508,7 @@ def resolve_cargo_workspace_members(
         possible_deps = prepare_possible_deps(
             package.get("dependencies", []),
             converter = cargo_metadata_dep_to_dep_dict,
+            skip_internal_rustc_placeholder_crates = skip_internal_rustc_placeholder_crates,
         )
 
         package_index = len(resolver_packages)
@@ -500,11 +555,12 @@ def resolve_cargo_workspace_members(
         for dep in package["dependencies"]:
             source = dep.get("source")
             dep_name = dep["name"]
-            dep_fq = fq_deps.get(dep_name)
+            dep_package = dep_package_name(dep)
+            dep_fq = select_package_fq_dep(dep, fq_deps)
             dep_version = None
             if dep_fq:
-                dep_version = dep_fq[len(dep_name) + 1:]
-            is_first_party_dep = not source and dep_version and (dep_name, dep_version) in workspace_member_keys
+                dep_version = dep_fq[len(dep_package) + 1:]
+            is_first_party_dep = not source and dep_version and (dep_package, dep_version) in workspace_member_keys
 
             if dep_fq and dep_fq not in feature_resolutions_by_fq_crate and allow_missing_resolved_deps:
                 continue
@@ -513,13 +569,13 @@ def resolve_cargo_workspace_members(
                 req = dep["req"]
                 fq = dep_fq
                 if req and fq:
-                    locked_version = fq[len(dep_name) + 1:]
+                    locked_version = fq[len(dep_package) + 1:]
                     if not select_matching_version(req, [locked_version]):
                         fail(("ERROR: Cargo.lock out of sync: %s requires %s %s but Cargo.lock has %s.\n\n" +
                               "If this is incorrect, please set `validate_lockfile = False` in `crate.from_cargo`\n" +
                               "and file a bug at https://github.com/hermeticbuild/rules_rs/issues/new") % (
                             package["name"],
-                            dep_name,
+                            dep_package,
                             req,
                             locked_version,
                         ))