From 113a277ef7908e82f60dfb3a51082cd25005bc7b Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Mon, 27 Oct 2025 23:39:10 +0000 Subject: [PATCH 01/20] Not finished openmp --- .bazelrc | 1 + .../sysroot_ubuntu18_x86_64.Dockerfile | 37 ++++---- cc/config/sysroot_ubuntu18_x86_64.BUILD | 11 +++ cc/deps/cc_toolchain_deps.bzl | 8 +- cc/deps/sysroot_linux.BUILD.tpl | 6 ++ cc/features/openmp_feature.bzl | 94 +++++++++++++++++++ cc/impls/linux_x86_64_linux_x86_64/BUILD | 31 ++++++ common/BUILD | 15 +++ 8 files changed, 182 insertions(+), 21 deletions(-) create mode 100644 cc/features/openmp_feature.bzl diff --git a/.bazelrc b/.bazelrc index 2dabc961..e9bbabbd 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,6 +4,7 @@ common:bzlmod --enable_bzlmod --noenable_workspace common --incompatible_enable_cc_toolchain_resolution # Only for bazel 6.5: C++ rules use platforms to select toolchains when you set this build --toolchain_resolution_debug=.* build --cxxopt=-std=c++17 +build --@rules_ml_toolchain//common:enable_openmp=True common:clang_local --noincompatible_enable_cc_toolchain_resolution build:clang_local --@rules_ml_toolchain//common:enable_hermetic_cc=False diff --git a/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile b/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile index 38307466..7892d752 100644 --- a/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile +++ b/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile @@ -9,23 +9,6 @@ # docker cp :/lib . # docker cp :/lib64 . -# Fix invalid links - -# Fix ld-linux-x86-64.so.2 -> /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 -# cd lib64/ -# sudo rm ./ld-linux-x86-64.so.2 -# sudo ln -s ../lib/x86_64-linux-gnu/ld-2.27.so ./ld-linux-x86-64.so.2 - -# Fix libdl.so -> /lib/x86_64-linux-gnu/libdl.so.2 -# cd usr/lib/x86_64-linux-gnu -# sudo rm ./libdl.so -# sudo ln -s ../../../lib/x86_64-linux-gnu/libdl.so.2 ./libdl.so - -# Fix libmvec.so -> /lib/x86_64-linux-gnu/libmvec.so.1 -# cd usr/lib/x86_64-linux-gnu/ -# rm ./libmvec.so -# sudo ln -s ../../../lib/x86_64-linux-gnu/libmvec.so.1 ./libmvec.so - FROM ubuntu:18.04 RUN apt-get update @@ -39,6 +22,26 @@ RUN rm -rf /usr/include/c++/7.5.0 RUN rm -rf /usr/include/x86_64-linux-gnu/c++/7 RUN rm -rf /usr/include/x86_64-linux-gnu/c++/7.5.0 +# Fix invalid links +WORKDIR /lib64 + +# Fix /lib64/ld-linux-x86-64.so.2 -> /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 +# cd lib64/ +# sudo ln -sfn ../lib/x86_64-linux-gnu/ld-2.27.so ./ld-linux-x86-64.so.2 +RUN ln -sfn ../lib/x86_64-linux-gnu/ld-2.27.so ./ld-linux-x86-64.so.2 + +WORKDIR /usr/lib/x86_64-linux-gnu + +# Fix /usr/lib/x86_64-linux-gnu/libdl.so -> /lib/x86_64-linux-gnu/libdl.so.2 +# cd usr/lib/x86_64-linux-gnu +# sudo ln -sfn ../../../lib/x86_64-linux-gnu/libdl.so.2 ./libdl.so +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libdl.so.2 ./libdl.so + +# Fix /usr/lib/x86_64-linux-gnu/libmvec.so -> /lib/x86_64-linux-gnu/libmvec.so.1 +# cd usr/lib/x86_64-linux-gnu/ +# sudo ln -sfn ../../../lib/x86_64-linux-gnu/libmvec.so.1 ./libmvec.so +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libmvec.so.1 ./libmvec.so + WORKDIR /root diff --git a/cc/config/sysroot_ubuntu18_x86_64.BUILD b/cc/config/sysroot_ubuntu18_x86_64.BUILD index a5cf62a4..7bf219d9 100644 --- a/cc/config/sysroot_ubuntu18_x86_64.BUILD +++ b/cc/config/sysroot_ubuntu18_x86_64.BUILD @@ -142,6 +142,17 @@ cc_toolchain_import( visibility = ["//visibility:public"], ) +# Application Programming Interface (API) for shared-memory parallel programming. +cc_toolchain_import( + name = "openmp", + additional_libs = glob([ + "usr/lib/x86_64-linux-gnu/libgomp*", + "usr/lib/x86_64-linux-gnu/libomp*", + ]), + shared_library = "usr/lib/x86_64-linux-gnu/libomp.so", + visibility = ["//visibility:public"], +) + cc_toolchain_import( name = "pthread", additional_libs = [ diff --git a/cc/deps/cc_toolchain_deps.bzl b/cc/deps/cc_toolchain_deps.bzl index 9f76ad64..606c9876 100644 --- a/cc/deps/cc_toolchain_deps.bzl +++ b/cc/deps/cc_toolchain_deps.bzl @@ -61,11 +61,11 @@ def cc_toolchain_deps(): # C++17, manylinux_2_27, gcc-8 mirrored_http_archive( name = "sysroot_linux_x86_64_glibc_2_27", - sha256 = "7d9300ec03d008e96f0098dfcd4e9974dd64f2d5fccdd0f1a2b2d4af63301a04", - mirrored_tar_sha256 = "8680378000cf63647db5421377fbdb6a4fe98e0207b87f7cfec51883bde587aa", - urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/ubuntu18_x86_64_sysroot_gcc8_patched-0.1.0.tar.xz"), + sha256 = "7c54a97a8f62d467d493f2942f9063d5ea3d6ec012fcecf4ba62ababe1889be5", + mirrored_tar_sha256 = "4d48dff78a648cf6a210f409e9b807a0bd212eda34708c9374251d8c8b6b4808", + urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/ubuntu18_x86_64_sysroot_gcc8_patched-0.1.1.tar.xz"), build_file = Label("//cc/config:sysroot_ubuntu18_x86_64.BUILD"), - strip_prefix = "ubuntu18_x86_64_sysroot_gcc8_patched-0.1.0", + strip_prefix = "ubuntu18_x86_64_sysroot_gcc8_patched-0.1.1", ) if "sysroot_linux_x86_64_glibc_2_31" not in native.existing_rules(): diff --git a/cc/deps/sysroot_linux.BUILD.tpl b/cc/deps/sysroot_linux.BUILD.tpl index 81964bb2..f55e8fd8 100644 --- a/cc/deps/sysroot_linux.BUILD.tpl +++ b/cc/deps/sysroot_linux.BUILD.tpl @@ -54,3 +54,9 @@ alias( actual = "@@%{sysroot_repo_name}//:pthread", visibility = ["//visibility:public"], ) + +alias( + name = "openmp", + actual = "@@%{sysroot_repo_name}//:openmp", + visibility = ["//visibility:public"], +) diff --git a/cc/features/openmp_feature.bzl b/cc/features/openmp_feature.bzl new file mode 100644 index 00000000..ff195704 --- /dev/null +++ b/cc/features/openmp_feature.bzl @@ -0,0 +1,94 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +load( + "@rules_cc//cc:action_names.bzl", + "ACTION_NAMES", + "ACTION_NAME_GROUPS", + "ALL_CC_COMPILE_ACTION_NAMES", + "ALL_CPP_COMPILE_ACTION_NAMES", + "CC_LINK_EXECUTABLE_ACTION_NAMES", + "DYNAMIC_LIBRARY_LINK_ACTION_NAMES", +) +load( + "@rules_cc//cc:cc_toolchain_config_lib.bzl", + "FeatureInfo", + "env_entry", + "env_set", + "feature", + "flag_group", + "flag_set", + _feature = "feature", +) +load( + "@rules_ml_toolchain//third_party/rules_cc_toolchain/features:cc_toolchain_import.bzl", + "CcToolchainImportInfo", +) + +def _import_openmp_feature_impl(ctx): + + print("_import_openmp_feature_impl: ==============================================") + #print("_import_openmp_feature_impl: toolchain_import.name =", ctx.attr.toolchain_import.name) + #print("_import_openmp_feature_impl: toolchain_import.package =", ctx.attr.toolchain_import.package) + #print("_import_openmp_feature_impl: toolchain_import.workspace_name =", ctx.attr.toolchain_import.workspace_name) + print("_import_openmp_feature_impl: workspace_root =", ctx.attr.toolchain_import.label.workspace_root) + print("_import_openmp_feature_impl: workspace_name =", ctx.attr.toolchain_import.label.workspace_name) + print("_import_openmp_feature_impl: package =", ctx.attr.toolchain_import.label.package) + print("_import_openmp_feature_impl: dir(...) =", dir(ctx.attr.toolchain_import)) + Label.relative(relName) + + toolchain_import_info = ctx.attr.toolchain_import[CcToolchainImportInfo] + + for path in toolchain_import_info.linking_context.runtime_paths.to_list(): + print("_import_openmp_feature_impl: ", path) + + linker_runtime_path_flags = depset([ + "-Wl,-rpath," + path + for path in toolchain_import_info + .linking_context.runtime_paths.to_list() + ]).to_list() + + print("_import_openmp_feature_impl: len(linker_runtime_path_flags) = ", len(linker_runtime_path_flags)) + flag_sets = [] + if linker_runtime_path_flags: + flag_sets.append(flag_set( + actions = CC_LINK_EXECUTABLE_ACTION_NAMES, + flag_groups = [ + flag_group( + flags = linker_runtime_path_flags, + ), + ], + )) + + library_feature = _feature( + name = ctx.label.name, + enabled = ctx.attr.enabled, + flag_sets = flag_sets, + implies = ctx.attr.implies, + provides = ctx.attr.provides, + ) + return [library_feature, ctx.attr.toolchain_import[DefaultInfo]] + +cc_toolchain_import_openmp_feature = rule( + _import_openmp_feature_impl, + attrs = { + "enabled": attr.bool(default = False), + "provides": attr.string_list(), + "requires": attr.string_list(), + "implies": attr.string_list(), + "toolchain_import": attr.label(), + }, + provides = [FeatureInfo, DefaultInfo], +) diff --git a/cc/impls/linux_x86_64_linux_x86_64/BUILD b/cc/impls/linux_x86_64_linux_x86_64/BUILD index 26d0e24c..106be92e 100644 --- a/cc/impls/linux_x86_64_linux_x86_64/BUILD +++ b/cc/impls/linux_x86_64_linux_x86_64/BUILD @@ -20,6 +20,7 @@ load("//third_party/rules_cc_toolchain:toolchain_config.bzl", "cc_toolchain_config") load("//third_party/rules_cc_toolchain/features:cc_toolchain_import.bzl", "cc_toolchain_import") load("//third_party/rules_cc_toolchain/features:features.bzl", "cc_toolchain_import_feature") +load("//cc/features:openmp_feature.bzl", "cc_toolchain_import_openmp_feature") package( default_visibility = [ @@ -61,6 +62,34 @@ cc_toolchain_import_feature( toolchain_import = ":imports", ) +#cc_toolchain_import( +# name = "openmp", + # features = select({ + # "//common:is_openmp_enabled": [ + # "openmp", + # ], + # "//conditions:default": [], + # }), +# deps = select({ +# "//common:is_openmp_enabled": [ +# "@sysroot_linux_x86_64//:openmp", +# ], +# "//conditions:default": [], +# }), +#) + +#cc_toolchain_import_feature( +# name = "openmp_feature", +# enabled = True, +# toolchain_import = ":openmp", +#) + +cc_toolchain_import_openmp_feature( + name = "openmp_feature", + enabled = True, + toolchain_import = "@sysroot_linux_x86_64//:openmp", +) + # buildifier: leave-alone filegroup( name = "all", @@ -93,6 +122,7 @@ filegroup( ":wrappers", "@llvm_linux_x86_64//:ld", "@sysroot_linux_x86_64//:startup_libs", + "@sysroot_linux_x86_64//:openmp", "@llvm_linux_x86_64//:distro_libs", ], ) @@ -116,6 +146,7 @@ cc_toolchain_config( # Hermetic libraries feature required before import. "//third_party/rules_cc_toolchain/features:hermetic_libraries", ":imports_feature", + ":openmp_feature", # Toolchain configuration "//third_party/rules_cc_toolchain/features:warnings", diff --git a/common/BUILD b/common/BUILD index 62393ea2..2c8da6a3 100644 --- a/common/BUILD +++ b/common/BUILD @@ -59,6 +59,21 @@ config_setting( }, ) +####################################################### +# Enable OpenMP + +bool_flag( + name = "enable_openmp", + build_setting_default = False, +) + +config_setting( + name = "is_openmp_enabled", + flag_values = { + ":enable_openmp": "True", + }, +) + ####################################################### # Enable CUDA support flags # From 1ce91911d874f2a2bf97618e3ff22f5d48809e27 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Wed, 29 Oct 2025 00:25:31 +0000 Subject: [PATCH 02/20] OpenMP in progress... --- .bazelrc | 2 +- MODULE.bazel | 2 +- WORKSPACE | 9 + .../sysroot_ubuntu18_x86_64.Dockerfile | 6 +- .../BUILD.bazel | 0 .../Dockerfile | 27 +++ .../sysroot_aarch64_ubuntu18.BUILD} | 23 +- cc/config/sysroot_ubuntu18_x86_64.BUILD | 27 ++- cc/deps/cc_toolchain_deps.bzl | 10 +- cc/deps/sysroot_linux.BUILD.tpl | 18 +- cc/features/openmp_feature.bzl | 50 ++++- cc/impls/linux_x86_64_linux_x86_64/BUILD | 5 +- cc/tests/cpu/BUILD | 9 +- third_party/protobuf.patch | 201 ++++++++++++++++++ 14 files changed, 344 insertions(+), 45 deletions(-) create mode 100644 cc/config/sysroot_aarch64_linux_glibc_2_27/BUILD.bazel create mode 100644 cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile rename cc/config/{sysroot_ubuntu18_aarch64.BUILD => sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD} (92%) create mode 100644 third_party/protobuf.patch diff --git a/.bazelrc b/.bazelrc index e9bbabbd..22501649 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,7 +4,7 @@ common:bzlmod --enable_bzlmod --noenable_workspace common --incompatible_enable_cc_toolchain_resolution # Only for bazel 6.5: C++ rules use platforms to select toolchains when you set this build --toolchain_resolution_debug=.* build --cxxopt=-std=c++17 -build --@rules_ml_toolchain//common:enable_openmp=True +#build --@rules_ml_toolchain//common:enable_openmp=True common:clang_local --noincompatible_enable_cc_toolchain_resolution build:clang_local --@rules_ml_toolchain//common:enable_hermetic_cc=False diff --git a/MODULE.bazel b/MODULE.bazel index 1144af20..cccbd169 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -78,7 +78,7 @@ use_repo( "llvm_linux_x86_64", "sysroot_darwin_aarch64", "sysroot_linux_aarch64", - "sysroot_linux_aarch64_glibc_2_27", + "sysroot_aarch64_linux_glibc_2_27", "sysroot_linux_aarch64_glibc_2_31", "sysroot_linux_x86_64", "sysroot_linux_x86_64_glibc_2_27", diff --git a/WORKSPACE b/WORKSPACE index 8c9ea742..00f516a7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -59,6 +59,15 @@ http_archive( urls = ["https://github.com/protocolbuffers/protobuf/archive/refs/tags/v5.28.3.zip"], ) +#http_archive( +# name = "com_google_protobuf", +# sha256 = "6e09bbc950ba60c3a7b30280210cd285af8d7d8ed5e0a6ed101c72aff22e8d88", +# strip_prefix = "protobuf-6.31.1", +# urls = ["https://github.com/protocolbuffers/protobuf/archive/refs/tags/v6.31.1.zip"], +# patches = ["//third_party:protobuf.patch"], +# patch_args = ["-p1"], +#) + load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") # Cross compilation error with older version. Details: https://github.com/bulletphysics/bullet3/issues/4607 diff --git a/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile b/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile index 7892d752..3c598e80 100644 --- a/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile +++ b/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile @@ -17,10 +17,10 @@ RUN apt-get -y install \ gcc-8 g++-8 \ libomp-dev -RUN rm -rf /usr/include/c++/7 RUN rm -rf /usr/include/c++/7.5.0 -RUN rm -rf /usr/include/x86_64-linux-gnu/c++/7 +RUN rm -rf /usr/include/c++/7 RUN rm -rf /usr/include/x86_64-linux-gnu/c++/7.5.0 +RUN rm -rf /usr/include/x86_64-linux-gnu/c++/7 # Fix invalid links WORKDIR /lib64 @@ -44,4 +44,4 @@ RUN ln -sfn ../../../lib/x86_64-linux-gnu/libmvec.so.1 ./libmvec.so WORKDIR /root - +# tar cf - tar_x86_64_- | xz -T8 -c > tar_x86_64_-.tar.xz diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/BUILD.bazel b/cc/config/sysroot_aarch64_linux_glibc_2_27/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile b/cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile new file mode 100644 index 00000000..756705bd --- /dev/null +++ b/cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile @@ -0,0 +1,27 @@ + +FROM arm64v8/ubuntu:bionic + +RUN apt-get update && apt-get -y install \ + build-essential \ + gcc-8 g++-8 \ + libomp-dev + +RUN rm -rf /usr/include/c++/7.5.0 +RUN rm -rf /usr/include/c++/7 +RUN rm -rf /usr/include/aarch64-linux-gnu/c++/7.5.0 +RUN rm -rf /usr/include/aarch64-linux-gnu/c++/7 + +WORKDIR /root + +# Create docker image from Dockerfile +# docker build -t sysroot_aarch64_ubuntu18:latest . + +# Run docker image +# docker run -it sysroot_aarch64_ubuntu18 + +# Copy needed directories from Docker image +# docker cp :/usr . +# docker cp :/lib . +# docker cp :/lib64 . + +# tar cf - sysroot_aarch64_ubuntu18_gcc8.4-X.Y.Z | xz -T8 -c > sysroot_aarch64_ubuntu18_gcc8.4-X.Y.Z.tar.xz \ No newline at end of file diff --git a/cc/config/sysroot_ubuntu18_aarch64.BUILD b/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD similarity index 92% rename from cc/config/sysroot_ubuntu18_aarch64.BUILD rename to cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD index f99ab379..0aac457f 100644 --- a/cc/config/sysroot_ubuntu18_aarch64.BUILD +++ b/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD @@ -182,14 +182,31 @@ cc_toolchain_import( ], ) -# This is a group of all the system libraries we need. The actual glibc library is split +# Application Programming Interface (API) for shared-memory parallel programming. +cc_toolchain_import( + name = "openmp", + additional_libs = glob([ + "usr/lib/aarch64-linux-gnu/libgomp*", + "usr/lib/aarch64-linux-gnu/libomp*", + ]), + visibility = ["//visibility:public"], +) + +cc_import( + name = "openmp_lib", + shared_library = "usr/lib/aarch64-linux-gnu/libomp.so", + visibility = ["//visibility:public"], +) + +# This is a group of all the system libraries we need. The actual essential libraries is split # out to fix link ordering problems that cause false undefined symbol positives. cc_toolchain_import( - name = "glibc", + name = "essential_libs", runtime_path = "/lib/aarch64-linux-gnu", visibility = ["//visibility:public"], deps = [ ":dynamic_linker", ":libc", + ":openmp", ], -) +) \ No newline at end of file diff --git a/cc/config/sysroot_ubuntu18_x86_64.BUILD b/cc/config/sysroot_ubuntu18_x86_64.BUILD index 7bf219d9..d060817d 100644 --- a/cc/config/sysroot_ubuntu18_x86_64.BUILD +++ b/cc/config/sysroot_ubuntu18_x86_64.BUILD @@ -142,17 +142,6 @@ cc_toolchain_import( visibility = ["//visibility:public"], ) -# Application Programming Interface (API) for shared-memory parallel programming. -cc_toolchain_import( - name = "openmp", - additional_libs = glob([ - "usr/lib/x86_64-linux-gnu/libgomp*", - "usr/lib/x86_64-linux-gnu/libomp*", - ]), - shared_library = "usr/lib/x86_64-linux-gnu/libomp.so", - visibility = ["//visibility:public"], -) - cc_toolchain_import( name = "pthread", additional_libs = [ @@ -208,3 +197,19 @@ cc_toolchain_import( ":libc", ], ) + +# Application Programming Interface (API) for shared-memory parallel programming. +cc_toolchain_import( + name = "openmp", + additional_libs = glob([ + "usr/lib/x86_64-linux-gnu/libgomp*", + "usr/lib/x86_64-linux-gnu/libomp*", + ]), + visibility = ["//visibility:public"], +) + +cc_import( + name = "openmp_lib", + shared_library = "usr/lib/x86_64-linux-gnu/libomp.so", + visibility = ["//visibility:public"], +) diff --git a/cc/deps/cc_toolchain_deps.bzl b/cc/deps/cc_toolchain_deps.bzl index 606c9876..ed2b0e52 100644 --- a/cc/deps/cc_toolchain_deps.bzl +++ b/cc/deps/cc_toolchain_deps.bzl @@ -106,11 +106,11 @@ def cc_toolchain_deps(): # C++17, manylinux_2_27, gcc-8 mirrored_http_archive( name = "sysroot_linux_aarch64_glibc_2_27", - sha256 = "31a285faccb6996c16acde8ef6841841d591f73196dc5b7bdd9cf55b7f0c35a1", - mirrored_tar_sha256 = "23989e1c4184b472ffcacf04cfeb7c9e108cb4126aeadb8a326597795b47f175", - urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/sysroot_aarch64_ubuntu18_gcc8.4-0.1.0.tar.xz"), - build_file = Label("//cc/config:sysroot_ubuntu18_aarch64.BUILD"), - strip_prefix = "sysroot_aarch64_ubuntu18_gcc8.4-0.1.0", + sha256 = "df19ac8b1415d1470ae721a3375e6111acdcf10756fc327bf78a0897249374e3", + mirrored_tar_sha256 = "f05a60f4cc1eaf6fa607cd59690218c3cbb8eddfab32e168e89c3cb4aaddbb22", + urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/sysroot_aarch64_ubuntu18_gcc8.4-0.2.0.tar.xz"), + build_file = Label("//cc/config/sysroot_aarch64_linux_glibc_2_27:sysroot_aarch64_ubuntu18.BUILD"), + strip_prefix = "sysroot_aarch64_ubuntu18_gcc8.4-0.2.0", ) if "sysroot_linux_aarch64_glibc_2_31" not in native.existing_rules(): diff --git a/cc/deps/sysroot_linux.BUILD.tpl b/cc/deps/sysroot_linux.BUILD.tpl index f55e8fd8..3358c0ba 100644 --- a/cc/deps/sysroot_linux.BUILD.tpl +++ b/cc/deps/sysroot_linux.BUILD.tpl @@ -43,12 +43,6 @@ alias( visibility = ["//visibility:public"], ) -alias( - name = "glibc", - actual = "@@%{sysroot_repo_name}//:glibc", - visibility = ["//visibility:public"], -) - alias( name = "pthread", actual = "@@%{sysroot_repo_name}//:pthread", @@ -60,3 +54,15 @@ alias( actual = "@@%{sysroot_repo_name}//:openmp", visibility = ["//visibility:public"], ) + +alias( + name = "openmp_lib", + actual = "@@%{sysroot_repo_name}//:openmp_lib", + visibility = ["//visibility:public"], +) + +alias( + name = "essential_libs", + actual = "@@%{sysroot_repo_name}//:essential_libs", + visibility = ["//visibility:public"], +) diff --git a/cc/features/openmp_feature.bzl b/cc/features/openmp_feature.bzl index ff195704..507ad83d 100644 --- a/cc/features/openmp_feature.bzl +++ b/cc/features/openmp_feature.bzl @@ -37,30 +37,58 @@ load( "CcToolchainImportInfo", ) +def _get_origin_path(lib_file): + relative_path = lib_file.short_path + print("_get_origin_path: relative_path = ", relative_path) + + repo_name = lib_file.owner.workspace_name + + path_in_repo = lib_file.path.removesuffix(lib_file.basename).removesuffix(repo_name + '/') + + origin_prefix = "$ORIGIN/../../../../../../" + print("_get_origin_path: Result: ", origin_prefix + "external/" + repo_name + "/" + path_in_repo) + return origin_prefix + "external/" + repo_name + "/" + path_in_repo + def _import_openmp_feature_impl(ctx): - print("_import_openmp_feature_impl: ==============================================") + print("_import_openmp_feature_impl: ==============================================", dir(ctx)) + print("_import_openmp_feature_impl: bin_dir = ", ctx.bin_dir.path) + print("_import_openmp_feature_impl: build_file_path = ", ctx.build_file_path) + print("_import_openmp_feature_impl: genfiles_dir = ", ctx.genfiles_dir) + print("_import_openmp_feature_impl: runfiles = ", dir(ctx.runfiles)) #print("_import_openmp_feature_impl: toolchain_import.name =", ctx.attr.toolchain_import.name) #print("_import_openmp_feature_impl: toolchain_import.package =", ctx.attr.toolchain_import.package) #print("_import_openmp_feature_impl: toolchain_import.workspace_name =", ctx.attr.toolchain_import.workspace_name) + print("_import_openmp_feature_impl: dir(...) =", dir(ctx.attr.toolchain_import)) print("_import_openmp_feature_impl: workspace_root =", ctx.attr.toolchain_import.label.workspace_root) print("_import_openmp_feature_impl: workspace_name =", ctx.attr.toolchain_import.label.workspace_name) print("_import_openmp_feature_impl: package =", ctx.attr.toolchain_import.label.package) - print("_import_openmp_feature_impl: dir(...) =", dir(ctx.attr.toolchain_import)) - Label.relative(relName) + print("_import_openmp_feature_impl: data_runfiles =", dir(ctx.attr.toolchain_import.data_runfiles)) + + runtime_paths = {} + for file in ctx.attr.toolchain_import.files.to_list(): + print("_import_openmp_feature_impl: file.basename =", file.basename) + print("_import_openmp_feature_impl: file.dirname =", file.dirname) + print("_import_openmp_feature_impl: file.extension =", file.extension) + print("_import_openmp_feature_impl: file.is_source =", file.is_source) + print("_import_openmp_feature_impl: file.path =", file.path) + print("_import_openmp_feature_impl: file.root =", dir(file.root)) + print("_import_openmp_feature_impl: file.short_path =", file.short_path) + print("_import_openmp_feature_impl: dir(file) =", dir(file)) + #print("_import_openmp_feature_impl: file.tree_relative_path =", file.tree_relative_path) + #print("_import_openmp_feature_impl: origin_path =", _get_origin_path(file)) - toolchain_import_info = ctx.attr.toolchain_import[CcToolchainImportInfo] - for path in toolchain_import_info.linking_context.runtime_paths.to_list(): - print("_import_openmp_feature_impl: ", path) + if file.path not in runtime_paths: + runtime_paths[file.dirname] = file + + toolchain_import_info = ctx.attr.toolchain_import[CcToolchainImportInfo] linker_runtime_path_flags = depset([ - "-Wl,-rpath," + path - for path in toolchain_import_info - .linking_context.runtime_paths.to_list() + "-Wl,-rpath," + _get_origin_path(runtime_paths[rpath]) + for rpath in runtime_paths ]).to_list() - print("_import_openmp_feature_impl: len(linker_runtime_path_flags) = ", len(linker_runtime_path_flags)) flag_sets = [] if linker_runtime_path_flags: flag_sets.append(flag_set( @@ -88,7 +116,7 @@ cc_toolchain_import_openmp_feature = rule( "provides": attr.string_list(), "requires": attr.string_list(), "implies": attr.string_list(), - "toolchain_import": attr.label(), + "toolchain_import": attr.label(providers = [CcToolchainImportInfo]), }, provides = [FeatureInfo, DefaultInfo], ) diff --git a/cc/impls/linux_x86_64_linux_x86_64/BUILD b/cc/impls/linux_x86_64_linux_x86_64/BUILD index 106be92e..1b5f1fa4 100644 --- a/cc/impls/linux_x86_64_linux_x86_64/BUILD +++ b/cc/impls/linux_x86_64_linux_x86_64/BUILD @@ -52,6 +52,7 @@ cc_toolchain_import( "@sysroot_linux_x86_64//:glibc", "@sysroot_linux_x86_64//:pthread", "@llvm_linux_x86_64//:libclang_rt", + #"@sysroot_linux_x86_64//:openmp", ], visibility = ["//visibility:public"], ) @@ -122,7 +123,7 @@ filegroup( ":wrappers", "@llvm_linux_x86_64//:ld", "@sysroot_linux_x86_64//:startup_libs", - "@sysroot_linux_x86_64//:openmp", + #"@sysroot_linux_x86_64//:openmp", "@llvm_linux_x86_64//:distro_libs", ], ) @@ -146,7 +147,7 @@ cc_toolchain_config( # Hermetic libraries feature required before import. "//third_party/rules_cc_toolchain/features:hermetic_libraries", ":imports_feature", - ":openmp_feature", + #":openmp_feature", # Toolchain configuration "//third_party/rules_cc_toolchain/features:warnings", diff --git a/cc/tests/cpu/BUILD b/cc/tests/cpu/BUILD index c0d85915..d1846b21 100644 --- a/cc/tests/cpu/BUILD +++ b/cc/tests/cpu/BUILD @@ -14,8 +14,9 @@ # ============================================================================== load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test") -load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") ############################################# # C @@ -106,6 +107,10 @@ cc_binary( name = "protobuf", srcs = ["protobuf.cc"], deps = [":protobuf_hello_lib"], + linkopts = [ + "-fopenmp", + "-lomp", + ], ) cc_test( diff --git a/third_party/protobuf.patch b/third_party/protobuf.patch new file mode 100644 index 00000000..e54be199 --- /dev/null +++ b/third_party/protobuf.patch @@ -0,0 +1,201 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -555,7 +555,8 @@ proto_lang_toolchain( + "//:cpp_features_proto", + "//:descriptor_proto", + ], +- command_line = "--cpp_out=$(OUT)", ++ command_line = "--cpp_out=dllexport_decl=PROTOBUF_EXPORT:$(OUT)", ++# command_line = "--cpp_out=$(OUT)", + runtime = "//src/google/protobuf", + visibility = ["//visibility:public"], + ) +diff --git a/build_defs/BUILD.bazel b/build_defs/BUILD.bazel +--- a/build_defs/BUILD.bazel ++++ b/build_defs/BUILD.bazel +@@ -1,6 +1,7 @@ + # Internal Starlark definitions for Protobuf. + + load("@bazel_skylib//lib:selects.bzl", "selects") ++load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") + load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") + load("//bazel:cc_proto_library.bzl", starlark_cc_proto_library = "cc_proto_library") + load(":cc_proto_blacklist_test.bzl", "cc_proto_blacklist_test") +@@ -13,6 +14,20 @@ package( + ], + ) + ++bool_flag( ++ name = "use_dlls", ++ build_setting_default = False, ++ visibility = ["//visibility:public"], ++) ++ ++config_setting( ++ name = "config_use_dlls", ++ flag_values = { ++ ":use_dlls": "True", ++ }, ++ visibility = ["//visibility:public"], ++) ++ + create_compiler_config_setting( + name = "config_msvc_cl", + value = "msvc-cl", +diff --git a/python/dist/system_python.bzl b/python/dist/system_python.bzl +--- a/python/dist/system_python.bzl ++++ b/python/dist/system_python.bzl +@@ -73,11 +73,10 @@ load("@bazel_skylib//lib:selects.bzl", "selects") + load("@bazel_skylib//rules:common_settings.bzl", "string_flag") + load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") + +-cc_library( +- name = "python_headers", +- hdrs = glob(["python/**/*.h"], allow_empty = True), +- includes = ["python"], +- visibility = ["//visibility:public"], ++alias( ++ name = "python_headers", ++ actual = "@rules_python//python/cc:current_py_cc_headers", ++ visibility = ["//visibility:public"], + ) + + string_flag( +@@ -219,7 +218,7 @@ def _system_python_impl(repository_ctx): + python3 = repository_ctx.which("python3") + python_version = _get_python_version(repository_ctx) + +- if path and python_version[0] == "3": ++ if False: + _populate_package(repository_ctx, path, python3, python_version) + else: + # buildifier: disable=print +diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py +--- a/python/google/protobuf/__init__.py ++++ b/python/google/protobuf/__init__.py +@@ -8,3 +8,9 @@ + # Copyright 2007 Google Inc. All Rights Reserved. + + __version__ = '6.31.1' ++ ++if __name__ != '__main__': ++ try: ++ __import__('pkg_resources').declare_namespace(__name__) ++ except ImportError: ++ __path__ = __import__('pkgutil').extend_path(__path__, __name__) +diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel +--- a/src/google/protobuf/BUILD.bazel ++++ b/src/google/protobuf/BUILD.bazel +@@ -525,6 +525,13 @@ cc_library( + "serial_arena.h", + "thread_safe_arena.h", + ], ++ local_defines = select({ ++ "//build_defs:config_use_dlls": [ ++ "PROTOBUF_USE_DLLS", ++ "LIBPROTOBUF_EXPORTS", ++ ], ++ "//conditions:default": [], ++ }), + strip_include_prefix = "/src", + visibility = [ + "//:__subpackages__", +@@ -657,7 +664,15 @@ cc_library( + "serial_arena.h", + "thread_safe_arena.h", + "wire_format_lite.h", ++ "port.h", + ], ++ local_defines = select({ ++ "//build_defs:config_use_dlls": [ ++ "PROTOBUF_USE_DLLS", ++ "LIBPROTOBUF_EXPORTS", ++ ], ++ "//conditions:default": [], ++ }), + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": [ +@@ -767,6 +782,13 @@ cc_library( + ], + hdrs = PROTOBUF_HEADERS, + copts = COPTS, ++ local_defines = select({ ++ "//build_defs:config_use_dlls": [ ++ "PROTOBUF_USE_DLLS", ++ "LIBPROTOBUF_EXPORTS", ++ ], ++ "//conditions:default": [], ++ }), + linkopts = LINK_OPTS, + strip_include_prefix = "/src", + visibility = [ +diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc +--- a/src/google/protobuf/arena.cc ++++ b/src/google/protobuf/arena.cc +@@ -547,7 +547,7 @@ ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() { + new internal::ThreadLocalStorage(); + return *thread_cache_->Get(); + } +-#elif defined(PROTOBUF_USE_DLLS) && defined(_WIN32) ++#elif defined(_WIN32) + ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() { + static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache; + return thread_cache; +diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel +--- a/src/google/protobuf/io/BUILD.bazel ++++ b/src/google/protobuf/io/BUILD.bazel +@@ -22,6 +22,13 @@ cc_library( + "zero_copy_stream_impl.h", + "zero_copy_stream_impl_lite.h", + ], ++ local_defines = select({ ++ "//build_defs:config_use_dlls": [ ++ "PROTOBUF_USE_DLLS", ++ "LIBPROTOBUF_EXPORTS", ++ ], ++ "//conditions:default": [], ++ }), + copts = COPTS, + strip_include_prefix = "/src", + deps = [ +diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h +--- a/src/google/protobuf/message_lite.h ++++ b/src/google/protobuf/message_lite.h +@@ -282,7 +282,11 @@ template + using MessageTraits = decltype(MessageTraitsImpl::value); + + struct EnumTraitsImpl { ++#ifdef __CUDACC__ ++ struct Undefined{}; ++#else + struct Undefined; ++#endif + template + static Undefined value; + }; +diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc +--- a/src/google/protobuf/port_def.inc ++++ b/src/google/protobuf/port_def.inc +@@ -421,7 +421,7 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), + #endif + + // Lexan sets both MSV_VER and clang, so handle it with the clang path. +-#if defined(_MSC_VER) && !defined(__clang__) ++#if defined(_MSC_VER) + // MSVC 17 currently seems to raise an error about constant-initialized pointers. + # if PROTOBUF_MSC_VER_MIN(1930) + # define PROTOBUF_CONSTINIT +diff --git a/src/google/protobuf/thread_safe_arena.h b/src/google/protobuf/thread_safe_arena.h +--- a/src/google/protobuf/thread_safe_arena.h ++++ b/src/google/protobuf/thread_safe_arena.h +@@ -249,7 +249,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { + // iOS does not support __thread keyword so we use a custom thread local + // storage class we implemented. + static ThreadCache& thread_cache(); +-#elif defined(PROTOBUF_USE_DLLS) && defined(_WIN32) ++#elif defined(_WIN32) + // Thread local variables cannot be exposed through MSVC DLL interface but we + // can wrap them in static functions. + static ThreadCache& thread_cache(); From e79780323ac82b916861fbedd5760f3b81b0e576 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Wed, 29 Oct 2025 01:33:52 +0000 Subject: [PATCH 03/20] Refactor BUILD file --- .../sysroot_aarch64_ubuntu18.BUILD | 8 +++--- cc/config/sysroot_darwin_aarch64.BUILD | 8 ++---- cc/config/sysroot_ubuntu18_x86_64_gcc9.BUILD | 10 +++++--- .../sysroot_ubuntu20_aarch64_gcc10.BUILD | 11 ++++---- cc/config/sysroot_ubuntu20_x86_64_gcc10.BUILD | 20 ++++++++++++--- cc/config/sysroot_ubuntu22_x86_64_gcc12.BUILD | 20 ++++++++++++--- .../BUILD.bazel | 0 .../Dockerfile} | 4 +-- .../sysroot_x86_64_ubuntu18.BUILD} | 25 +++++++++++-------- cc/deps/cc_toolchain_deps.bzl | 2 +- cc/deps/sysroot_linux.BUILD.tpl | 22 ++-------------- cc/impls/darwin_aarch64_darwin_aarch64/BUILD | 3 +-- cc/impls/linux_aarch64_linux_aarch64/BUILD | 3 +-- .../linux_aarch64_linux_aarch64_cuda/BUILD | 3 +-- cc/impls/linux_x86_64_darwin_aarch64/BUILD | 3 +-- cc/impls/linux_x86_64_linux_aarch64/BUILD | 3 +-- cc/impls/linux_x86_64_linux_x86_64/BUILD | 12 +-------- cc/impls/linux_x86_64_linux_x86_64_cuda/BUILD | 3 +-- cc/impls/linux_x86_64_linux_x86_64_sycl/BUILD | 3 +-- 19 files changed, 77 insertions(+), 86 deletions(-) create mode 100644 cc/config/sysroot_x86_64_linux_glibc_2_27/BUILD.bazel rename cc/config/{sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile => sysroot_x86_64_linux_glibc_2_27/Dockerfile} (86%) rename cc/config/{sysroot_ubuntu18_x86_64.BUILD => sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD} (98%) diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD b/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD index 0aac457f..ebea5bc0 100644 --- a/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD +++ b/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD @@ -201,12 +201,12 @@ cc_import( # This is a group of all the system libraries we need. The actual essential libraries is split # out to fix link ordering problems that cause false undefined symbol positives. cc_toolchain_import( - name = "essential_libs", - runtime_path = "/lib/aarch64-linux-gnu", - visibility = ["//visibility:public"], + name = "libs", deps = [ ":dynamic_linker", ":libc", ":openmp", + ":pthread", ], -) \ No newline at end of file + visibility = ["//visibility:public"], +) diff --git a/cc/config/sysroot_darwin_aarch64.BUILD b/cc/config/sysroot_darwin_aarch64.BUILD index ae220e56..02a1471e 100644 --- a/cc/config/sysroot_darwin_aarch64.BUILD +++ b/cc/config/sysroot_darwin_aarch64.BUILD @@ -133,13 +133,8 @@ cc_toolchain_import( # This is a group of all the system libraries we need. The actual glibc library is split # out to fix link ordering problems that cause false undefined symbol positives. cc_toolchain_import( - name = "glibc", + name = "libs", #runtime_path = "usr/lib", - #target_compatible_with = select({ - # "@platforms//os:macos": ["@platforms//cpu:aarch64"], - # "//conditions:default": ["@platforms//:incompatible"], - #}), - visibility = ["//visibility:public"], deps = [ ":system", ":libm", @@ -147,4 +142,5 @@ cc_toolchain_import( ":objc", ":core_foundation", ], + visibility = ["//visibility:public"], ) diff --git a/cc/config/sysroot_ubuntu18_x86_64_gcc9.BUILD b/cc/config/sysroot_ubuntu18_x86_64_gcc9.BUILD index de1e3afe..1f2a3b5e 100644 --- a/cc/config/sysroot_ubuntu18_x86_64_gcc9.BUILD +++ b/cc/config/sysroot_ubuntu18_x86_64_gcc9.BUILD @@ -169,22 +169,24 @@ cc_toolchain_import( ], shared_library = "usr/lib/x86_64-linux-gnu/libc.so", static_library = "usr/lib/x86_64-linux-gnu/libc.a", - visibility = ["//visibility:public"], deps = [ ":gcc", ":math", ":stdc++", ":rt", ], + visibility = ["//visibility:public"], ) -# This is a group of all the system libraries we need. The actual glibc library is split +# This is a group of all the system libraries we need. The actual essential libraries is split # out to fix link ordering problems that cause false undefined symbol positives. cc_toolchain_import( - name = "glibc", - visibility = ["//visibility:public"], + name = "libs", deps = [ ":dynamic_linker", ":libc", + ":openmp", + ":pthread", ], + visibility = ["//visibility:public"], ) diff --git a/cc/config/sysroot_ubuntu20_aarch64_gcc10.BUILD b/cc/config/sysroot_ubuntu20_aarch64_gcc10.BUILD index 39eb74d6..98c861c3 100644 --- a/cc/config/sysroot_ubuntu20_aarch64_gcc10.BUILD +++ b/cc/config/sysroot_ubuntu20_aarch64_gcc10.BUILD @@ -163,23 +163,24 @@ cc_toolchain_import( # "@platforms//os:linux": ["@platforms//cpu:aarch64"], # "//conditions:default": ["@platforms//:incompatible"], #}), - visibility = ["//visibility:public"], deps = [ ":gcc", ":math", ":stdc++", ":rt", ], + visibility = ["//visibility:public"], ) -# This is a group of all the system libraries we need. The actual glibc library is split +# This is a group of all the system libraries we need. The actual essential libraries is split # out to fix link ordering problems that cause false undefined symbol positives. cc_toolchain_import( - name = "glibc", - runtime_path = "/lib/aarch64-linux-gnu", - visibility = ["//visibility:public"], + name = "libs", deps = [ ":dynamic_linker", ":libc", + ":openmp", + ":pthread", ], + visibility = ["//visibility:public"], ) diff --git a/cc/config/sysroot_ubuntu20_x86_64_gcc10.BUILD b/cc/config/sysroot_ubuntu20_x86_64_gcc10.BUILD index 41fa411b..801a8ea7 100644 --- a/cc/config/sysroot_ubuntu20_x86_64_gcc10.BUILD +++ b/cc/config/sysroot_ubuntu20_x86_64_gcc10.BUILD @@ -167,22 +167,34 @@ cc_toolchain_import( ], shared_library = "usr/lib/x86_64-linux-gnu/libc.so", static_library = "usr/lib/x86_64-linux-gnu/libc.a", - visibility = ["//visibility:public"], deps = [ ":gcc", ":math", ":stdc++", ":rt", ], + visibility = ["//visibility:public"], ) -# This is a group of all the system libraries we need. The actual glibc library is split -# out to fix link ordering problems that cause false undefined symbol positives. +# Application Programming Interface (API) for shared-memory parallel programming. cc_toolchain_import( - name = "glibc", + name = "openmp", + additional_libs = glob([ + "usr/lib/x86_64-linux-gnu/libgomp*", + "usr/lib/x86_64-linux-gnu/libomp*", + ]), visibility = ["//visibility:public"], +) + +# This is a group of all the system libraries we need. The actual essential libraries is split +# out to fix link ordering problems that cause false undefined symbol positives. +cc_toolchain_import( + name = "libs", deps = [ ":dynamic_linker", ":libc", + ":openmp", + ":pthread", ], + visibility = ["//visibility:public"], ) diff --git a/cc/config/sysroot_ubuntu22_x86_64_gcc12.BUILD b/cc/config/sysroot_ubuntu22_x86_64_gcc12.BUILD index 245bce0f..4ce3ba6c 100644 --- a/cc/config/sysroot_ubuntu22_x86_64_gcc12.BUILD +++ b/cc/config/sysroot_ubuntu22_x86_64_gcc12.BUILD @@ -162,22 +162,34 @@ cc_toolchain_import( ], shared_library = "usr/lib/x86_64-linux-gnu/libc.so", static_library = "usr/lib/x86_64-linux-gnu/libc.a", - visibility = ["//visibility:public"], deps = [ ":gcc", ":math", ":stdc++", ":rt", ], + visibility = ["//visibility:public"], ) -# This is a group of all the system libraries we need. The actual glibc library is split -# out to fix link ordering problems that cause false undefined symbol positives. +# Application Programming Interface (API) for shared-memory parallel programming. cc_toolchain_import( - name = "glibc", + name = "openmp", + additional_libs = glob([ + "usr/lib/x86_64-linux-gnu/libgomp*", + "usr/lib/x86_64-linux-gnu/libomp*", + ]), visibility = ["//visibility:public"], +) + +# This is a group of all the system libraries we need. The actual essential libraries is split +# out to fix link ordering problems that cause false undefined symbol positives. +cc_toolchain_import( + name = "libs", deps = [ ":dynamic_linker", ":libc", + ":openmp", + ":pthread", ], + visibility = ["//visibility:public"], ) diff --git a/cc/config/sysroot_x86_64_linux_glibc_2_27/BUILD.bazel b/cc/config/sysroot_x86_64_linux_glibc_2_27/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile b/cc/config/sysroot_x86_64_linux_glibc_2_27/Dockerfile similarity index 86% rename from cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile rename to cc/config/sysroot_x86_64_linux_glibc_2_27/Dockerfile index 3c598e80..4afb94b8 100644 --- a/cc/config/sysroot-dockerfiles/sysroot_ubuntu18_x86_64.Dockerfile +++ b/cc/config/sysroot_x86_64_linux_glibc_2_27/Dockerfile @@ -1,5 +1,5 @@ # Create docker image from Dockerfile -# docker build -t sysroot_ubuntu18_x86_64:latest -f ./sysroot_ubuntu18_x86_64.Dockerfile . +# docker build -t sysroot_ubuntu18_x86_64:latest -f . # Run docker image # docker run -it sysroot_ubuntu18_x86_64 @@ -44,4 +44,4 @@ RUN ln -sfn ../../../lib/x86_64-linux-gnu/libmvec.so.1 ./libmvec.so WORKDIR /root -# tar cf - tar_x86_64_- | xz -T8 -c > tar_x86_64_-.tar.xz +# tar cf - sysroot_x86_64_ubuntu18_gcc8.4-X.Y.Z | xz -T8 -c > sysroot_x86_64_ubuntu18_gcc8.4-X.Y.Z.tar.xz diff --git a/cc/config/sysroot_ubuntu18_x86_64.BUILD b/cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD similarity index 98% rename from cc/config/sysroot_ubuntu18_x86_64.BUILD rename to cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD index d060817d..2ea50c91 100644 --- a/cc/config/sysroot_ubuntu18_x86_64.BUILD +++ b/cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD @@ -1,4 +1,5 @@ # Copyright 2025 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -187,17 +188,6 @@ cc_toolchain_import( ], ) -# This is a group of all the system libraries we need. The actual glibc library is split -# out to fix link ordering problems that cause false undefined symbol positives. -cc_toolchain_import( - name = "glibc", - visibility = ["//visibility:public"], - deps = [ - ":dynamic_linker", - ":libc", - ], -) - # Application Programming Interface (API) for shared-memory parallel programming. cc_toolchain_import( name = "openmp", @@ -213,3 +203,16 @@ cc_import( shared_library = "usr/lib/x86_64-linux-gnu/libomp.so", visibility = ["//visibility:public"], ) + +# This is a group of all the system libraries we need. The actual essential libraries is split +# out to fix link ordering problems that cause false undefined symbol positives. +cc_toolchain_import( + name = "libs", + deps = [ + ":dynamic_linker", + ":libc", + ":openmp", + ":pthread", + ], + visibility = ["//visibility:public"], +) diff --git a/cc/deps/cc_toolchain_deps.bzl b/cc/deps/cc_toolchain_deps.bzl index ed2b0e52..f2849872 100644 --- a/cc/deps/cc_toolchain_deps.bzl +++ b/cc/deps/cc_toolchain_deps.bzl @@ -64,7 +64,7 @@ def cc_toolchain_deps(): sha256 = "7c54a97a8f62d467d493f2942f9063d5ea3d6ec012fcecf4ba62ababe1889be5", mirrored_tar_sha256 = "4d48dff78a648cf6a210f409e9b807a0bd212eda34708c9374251d8c8b6b4808", urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/ubuntu18_x86_64_sysroot_gcc8_patched-0.1.1.tar.xz"), - build_file = Label("//cc/config:sysroot_ubuntu18_x86_64.BUILD"), + build_file = Label("//cc/config/sysroot_x86_64_linux_glibc_2_27:sysroot_x86_64_ubuntu18.BUILD"), strip_prefix = "ubuntu18_x86_64_sysroot_gcc8_patched-0.1.1", ) diff --git a/cc/deps/sysroot_linux.BUILD.tpl b/cc/deps/sysroot_linux.BUILD.tpl index 3358c0ba..6e99d237 100644 --- a/cc/deps/sysroot_linux.BUILD.tpl +++ b/cc/deps/sysroot_linux.BUILD.tpl @@ -44,25 +44,7 @@ alias( ) alias( - name = "pthread", - actual = "@@%{sysroot_repo_name}//:pthread", - visibility = ["//visibility:public"], -) - -alias( - name = "openmp", - actual = "@@%{sysroot_repo_name}//:openmp", - visibility = ["//visibility:public"], -) - -alias( - name = "openmp_lib", - actual = "@@%{sysroot_repo_name}//:openmp_lib", - visibility = ["//visibility:public"], -) - -alias( - name = "essential_libs", - actual = "@@%{sysroot_repo_name}//:essential_libs", + name = "libs", + actual = "@@%{sysroot_repo_name}//:libs", visibility = ["//visibility:public"], ) diff --git a/cc/impls/darwin_aarch64_darwin_aarch64/BUILD b/cc/impls/darwin_aarch64_darwin_aarch64/BUILD index 2f2fb10e..498235bc 100644 --- a/cc/impls/darwin_aarch64_darwin_aarch64/BUILD +++ b/cc/impls/darwin_aarch64_darwin_aarch64/BUILD @@ -43,8 +43,7 @@ cc_toolchain_import( "@sysroot_darwin_aarch64//:includes_c", "@llvm_darwin_aarch64//:includes", "@sysroot_darwin_aarch64//:includes_system", - "@sysroot_darwin_aarch64//:glibc", - "@sysroot_darwin_aarch64//:pthread", + "@sysroot_darwin_aarch64//:libs", ], visibility = ["//visibility:public"], ) diff --git a/cc/impls/linux_aarch64_linux_aarch64/BUILD b/cc/impls/linux_aarch64_linux_aarch64/BUILD index 63fe51a8..f88bfc38 100644 --- a/cc/impls/linux_aarch64_linux_aarch64/BUILD +++ b/cc/impls/linux_aarch64_linux_aarch64/BUILD @@ -54,8 +54,7 @@ cc_toolchain_import( "@sysroot_linux_aarch64//:includes_c", "@llvm_linux_aarch64//:includes", "@sysroot_linux_aarch64//:includes_system", - "@sysroot_linux_aarch64//:glibc", - "@sysroot_linux_aarch64//:pthread", + "@sysroot_linux_aarch64//:libs", ], visibility = ["//visibility:public"], ) diff --git a/cc/impls/linux_aarch64_linux_aarch64_cuda/BUILD b/cc/impls/linux_aarch64_linux_aarch64_cuda/BUILD index 1c70dfcd..20db98fc 100644 --- a/cc/impls/linux_aarch64_linux_aarch64_cuda/BUILD +++ b/cc/impls/linux_aarch64_linux_aarch64_cuda/BUILD @@ -96,8 +96,7 @@ cc_toolchain_import( "@sysroot_linux_aarch64//:includes_c", "@llvm_linux_aarch64//:includes", "@sysroot_linux_aarch64//:includes_system", - "@sysroot_linux_aarch64//:glibc", - "@sysroot_linux_aarch64//:pthread", + "@sysroot_linux_aarch64//:libs", "@llvm_linux_aarch64//:libclang_rt", ], visibility = ["//visibility:public"], diff --git a/cc/impls/linux_x86_64_darwin_aarch64/BUILD b/cc/impls/linux_x86_64_darwin_aarch64/BUILD index e9c41d49..53bc9172 100644 --- a/cc/impls/linux_x86_64_darwin_aarch64/BUILD +++ b/cc/impls/linux_x86_64_darwin_aarch64/BUILD @@ -43,8 +43,7 @@ cc_toolchain_import( "@sysroot_darwin_aarch64//:includes_c", "@llvm_linux_x86_64//:includes", "@sysroot_darwin_aarch64//:includes_system", - "@sysroot_darwin_aarch64//:glibc", - "@sysroot_darwin_aarch64//:pthread", + "@sysroot_darwin_aarch64//:libs", ], visibility = ["//visibility:public"], ) diff --git a/cc/impls/linux_x86_64_linux_aarch64/BUILD b/cc/impls/linux_x86_64_linux_aarch64/BUILD index c11a767f..2f77dcdf 100644 --- a/cc/impls/linux_x86_64_linux_aarch64/BUILD +++ b/cc/impls/linux_x86_64_linux_aarch64/BUILD @@ -54,8 +54,7 @@ cc_toolchain_import( "@sysroot_linux_aarch64//:includes_c", "@llvm_linux_x86_64//:includes", "@sysroot_linux_aarch64//:includes_system", - "@sysroot_linux_aarch64//:glibc", - "@sysroot_linux_aarch64//:pthread", + "@sysroot_linux_aarch64//:libs", ], visibility = ["//visibility:public"], ) diff --git a/cc/impls/linux_x86_64_linux_x86_64/BUILD b/cc/impls/linux_x86_64_linux_x86_64/BUILD index 1b5f1fa4..31e1dc7f 100644 --- a/cc/impls/linux_x86_64_linux_x86_64/BUILD +++ b/cc/impls/linux_x86_64_linux_x86_64/BUILD @@ -49,10 +49,8 @@ cc_toolchain_import( "@sysroot_linux_x86_64//:includes_c", "@llvm_linux_x86_64//:includes", "@sysroot_linux_x86_64//:includes_system", - "@sysroot_linux_x86_64//:glibc", - "@sysroot_linux_x86_64//:pthread", + "@sysroot_linux_x86_64//:libs", "@llvm_linux_x86_64//:libclang_rt", - #"@sysroot_linux_x86_64//:openmp", ], visibility = ["//visibility:public"], ) @@ -85,12 +83,6 @@ cc_toolchain_import_feature( # toolchain_import = ":openmp", #) -cc_toolchain_import_openmp_feature( - name = "openmp_feature", - enabled = True, - toolchain_import = "@sysroot_linux_x86_64//:openmp", -) - # buildifier: leave-alone filegroup( name = "all", @@ -123,7 +115,6 @@ filegroup( ":wrappers", "@llvm_linux_x86_64//:ld", "@sysroot_linux_x86_64//:startup_libs", - #"@sysroot_linux_x86_64//:openmp", "@llvm_linux_x86_64//:distro_libs", ], ) @@ -147,7 +138,6 @@ cc_toolchain_config( # Hermetic libraries feature required before import. "//third_party/rules_cc_toolchain/features:hermetic_libraries", ":imports_feature", - #":openmp_feature", # Toolchain configuration "//third_party/rules_cc_toolchain/features:warnings", diff --git a/cc/impls/linux_x86_64_linux_x86_64_cuda/BUILD b/cc/impls/linux_x86_64_linux_x86_64_cuda/BUILD index dbe4766b..4cfde5d1 100644 --- a/cc/impls/linux_x86_64_linux_x86_64_cuda/BUILD +++ b/cc/impls/linux_x86_64_linux_x86_64_cuda/BUILD @@ -98,8 +98,7 @@ cc_toolchain_import( "@sysroot_linux_x86_64//:includes_c", "@llvm_linux_x86_64//:includes", "@sysroot_linux_x86_64//:includes_system", - "@sysroot_linux_x86_64//:glibc", - "@sysroot_linux_x86_64//:pthread", + "@sysroot_linux_x86_64//:libs", "@llvm_linux_x86_64//:libclang_rt", ], visibility = ["//visibility:public"], diff --git a/cc/impls/linux_x86_64_linux_x86_64_sycl/BUILD b/cc/impls/linux_x86_64_linux_x86_64_sycl/BUILD index 5342ac34..5660d782 100644 --- a/cc/impls/linux_x86_64_linux_x86_64_sycl/BUILD +++ b/cc/impls/linux_x86_64_linux_x86_64_sycl/BUILD @@ -77,8 +77,7 @@ cc_toolchain_import( "@sysroot_linux_x86_64//:includes_c", "@oneapi//:includes", "@sysroot_linux_x86_64//:includes_system", - "@sysroot_linux_x86_64//:glibc", - "@sysroot_linux_x86_64//:pthread", + "@sysroot_linux_x86_64//:libs", "@oneapi//:core", "@oneapi//:mkl", "@oneapi//:libclang_rt", From dac2283c51d592a333af8ee52acb12dc5fd381b6 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Wed, 29 Oct 2025 19:01:05 +0000 Subject: [PATCH 04/20] Make litert available in dynamic section --- .../sysroot_aarch64_ubuntu18.BUILD | 4 ++-- .../sysroot_x86_64_ubuntu18.BUILD | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD b/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD index ebea5bc0..99e07085 100644 --- a/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD +++ b/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD @@ -153,9 +153,9 @@ cc_toolchain_import( additional_libs = [ "lib/aarch64-linux-gnu/librt-{glibc_version}.so".format(glibc_version = GLIBC_VERSION), "lib/aarch64-linux-gnu/librt.so.1", - "usr/lib/aarch64-linux-gnu/librt.so", - "usr/lib/aarch64-linux-gnu/librt.a", ], + shared_library = "usr/lib/aarch64-linux-gnu/librt.so", + static_library = "usr/lib/aarch64-linux-gnu/librt.a", visibility = ["//visibility:private"], ) diff --git a/cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD b/cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD index 2ea50c91..e8d12779 100644 --- a/cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD +++ b/cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD @@ -163,9 +163,9 @@ cc_toolchain_import( additional_libs = [ "lib/x86_64-linux-gnu/librt-{glibc_version}.so".format(glibc_version = GLIBC_VERSION), "lib/x86_64-linux-gnu/librt.so.1", - "usr/lib/x86_64-linux-gnu/librt.so", - "usr/lib/x86_64-linux-gnu/librt.a", ], + shared_library = "usr/lib/x86_64-linux-gnu/librt.so", + static_library = "usr/lib/x86_64-linux-gnu/librt.a", visibility = ["//visibility:private"], ) From 63740b4ccfdb929ab6205331b4032f0d1e1af9f1 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Wed, 29 Oct 2025 23:58:11 +0000 Subject: [PATCH 05/20] Create bash script for sysroot creation --- .../Dockerfile | 52 ++++++++++++++----- .../create-sysroot.sh | 30 +++++++++++ 2 files changed, 69 insertions(+), 13 deletions(-) create mode 100644 cc/config/sysroot_aarch64_linux_glibc_2_27/create-sysroot.sh diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile b/cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile index 756705bd..6bbaea0d 100644 --- a/cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile +++ b/cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile @@ -10,18 +10,44 @@ RUN rm -rf /usr/include/c++/7.5.0 RUN rm -rf /usr/include/c++/7 RUN rm -rf /usr/include/aarch64-linux-gnu/c++/7.5.0 RUN rm -rf /usr/include/aarch64-linux-gnu/c++/7 +RUN rm -rf /usr/lib/gcc/aarch64-linux-gnu/7 + +# Replace absulte links by relative in /usr/lib/aarch64-linux-gnu/ directory +WORKDIR /usr/lib/aarch64-linux-gnu + +# /usr/lib/aarch64-linux-gnu/libBrokenLocale.so -> /lib/aarch64-linux-gnu/libBrokenLocale.so.1 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libBrokenLocale.so.1 ./libBrokenLocale.so +# /usr/lib/aarch64-linux-gnu/libanl.so -> /lib/aarch64-linux-gnu/libanl.so.1 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libanl.so.1 ./libanl.so +# /usr/lib/aarch64-linux-gnu/libcidn.so -> /lib/aarch64-linux-gnu/libcidn.so.1 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libcidn.so.1 ./libcidn.so +# /usr/lib/aarch64-linux-gnu/libcrypt.so -> /lib/aarch64-linux-gnu/libcrypt.so.1 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libcrypt.so.1 ./libcrypt.so +# /usr/lib/aarch64-linux-gnu/libdl.so -> /lib/aarch64-linux-gnu/libdl.so.2 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libdl.so.2 ./libdl.so +# /usr/lib/aarch64-linux-gnu/libm.so -> /lib/aarch64-linux-gnu/libm.so.6 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libm.so.6 ./libm.so +# /usr/lib/aarch64-linux-gnu/libnsl.so -> /lib/aarch64-linux-gnu/libnsl.so.1 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libnsl.so.1 ./libnsl.so +# /usr/lib/aarch64-linux-gnu/libnss_compat.so -> /lib/aarch64-linux-gnu/libnss_compat.so.2 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libnss_compat.so.2 ./libnss_compat.so +# /usr/lib/aarch64-linux-gnu/libnss_dns.so -> /lib/aarch64-linux-gnu/libnss_dns.so.2 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libnss_dns.so.2 ./libnss_dns.so +# /usr/lib/aarch64-linux-gnu/libnss_files.so -> /lib/aarch64-linux-gnu/libnss_files.so.2 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libnss_files.so.2 ./libnss_files.so +# /usr/lib/aarch64-linux-gnu/libnss_hesiod.so -> /lib/aarch64-linux-gnu/libnss_hesiod.so.2 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libnss_hesiod.so.2 ./libnss_hesiod.so +# /usr/lib/aarch64-linux-gnu/libnss_nis.so -> /lib/aarch64-linux-gnu/libnss_nis.so.2 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libnss_nis.so.2 ./libnss_nis.so +# /usr/lib/aarch64-linux-gnu/libnss_nisplus.so -> /lib/aarch64-linux-gnu/libnss_nisplus.so.2 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libnss_nisplus.so.2 ./libnss_nisplus.so +# /usr/lib/aarch64-linux-gnu/libresolv.so -> /lib/aarch64-linux-gnu/libresolv.so.2 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libresolv.so.2 ./libresolv.so +# /usr/lib/aarch64-linux-gnu/librt.so -> /lib/aarch64-linux-gnu/librt.so.1 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/librt.so.1 ./librt.so +# /usr/lib/aarch64-linux-gnu/libthread_db.so -> /lib/aarch64-linux-gnu/libthread_db.so.1 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libthread_db.so.1 ./libthread_db.so +# /usr/lib/aarch64-linux-gnu/libutil.so -> /lib/aarch64-linux-gnu/libutil.so.1 +RUN ln -sfn ../../../lib/aarch64-linux-gnu/libutil.so.1 ./libutil.so WORKDIR /root - -# Create docker image from Dockerfile -# docker build -t sysroot_aarch64_ubuntu18:latest . - -# Run docker image -# docker run -it sysroot_aarch64_ubuntu18 - -# Copy needed directories from Docker image -# docker cp :/usr . -# docker cp :/lib . -# docker cp :/lib64 . - -# tar cf - sysroot_aarch64_ubuntu18_gcc8.4-X.Y.Z | xz -T8 -c > sysroot_aarch64_ubuntu18_gcc8.4-X.Y.Z.tar.xz \ No newline at end of file diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/create-sysroot.sh b/cc/config/sysroot_aarch64_linux_glibc_2_27/create-sysroot.sh new file mode 100644 index 00000000..59de59ce --- /dev/null +++ b/cc/config/sysroot_aarch64_linux_glibc_2_27/create-sysroot.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +ARCH_NAME=sysroot_aarch64_ubuntu18_gcc8.4-0.2.0 + +# Remove old files and image +rm -f ./$ARCH_NAME/$ARCH_NAME.tar.xz +rm -rf ./$ARCH_NAME +docker rm -f u18sysroot +docker rmi -f $ARCH_NAME:latest + +# Create docker image from Dockerfile +docker build -t $ARCH_NAME:latest . + +mkdir ./$ARCH_NAME + +# Run docker image +docker run -d --name u18sysroot $ARCH_NAME:latest bash -c "sleep 20" + +# Copy needed directories from Docker image +docker cp u18sysroot:/lib ./$ARCH_NAME/ +docker cp u18sysroot:/usr ./$ARCH_NAME/ +rm -rf ./$ARCH_NAME/lib/cpp +rm -rf ./$ARCH_NAME/usr/bin +rm -rf ./$ARCH_NAME/usr/games +rm -rf ./$ARCH_NAME/usr/sbin +rm -rf ./$ARCH_NAME/usr/share +rm -rf ./$ARCH_NAME/usr/src + +echo "Creating $ARCH_NAME.tar.xz archive..." +tar cf - $ARCH_NAME | xz -T8 -c > $ARCH_NAME.tar.xz From 44606b514f25d34a8a7a28b2ad0a27567b314211 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Wed, 29 Oct 2025 23:58:46 +0000 Subject: [PATCH 06/20] Remove runtime paths for Linux aarch64 sysroot --- .../sysroot_aarch64_ubuntu18.BUILD | 2 -- 1 file changed, 2 deletions(-) diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD b/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD index 99e07085..8da05170 100644 --- a/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD +++ b/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD @@ -119,7 +119,6 @@ cc_toolchain_import( "lib/aarch64-linux-gnu/ld-linux-aarch64.so.1", "lib/aarch64-linux-gnu/ld-{glibc_version}.so".format(glibc_version = GLIBC_VERSION), ], - runtime_path = "/lib64", shared_library = "usr/lib/aarch64-linux-gnu/libdl.so", static_library = "usr/lib/aarch64-linux-gnu/libdl.a", deps = [":libc"], @@ -165,7 +164,6 @@ cc_toolchain_import( "lib/aarch64-linux-gnu/libc.so.6", "usr/lib/aarch64-linux-gnu/libc_nonshared.a", ], - runtime_path = "/usr/lib/gcc/aarch64-linux-gnu/{gcc_version}".format(gcc_version = GCC_VERSION), shared_library = "usr/lib/aarch64-linux-gnu/libc.so", static_library = "usr/lib/aarch64-linux-gnu/libc.a", #target_compatible_with = select({ From 8761a134a0133384c6e1e6f7ce72fd7d52610f95 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Thu, 30 Oct 2025 00:09:58 +0000 Subject: [PATCH 07/20] Update sysroot_linux_aarch64_glibc_2_27 --- cc/deps/cc_toolchain_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cc/deps/cc_toolchain_deps.bzl b/cc/deps/cc_toolchain_deps.bzl index f2849872..bb0a3a33 100644 --- a/cc/deps/cc_toolchain_deps.bzl +++ b/cc/deps/cc_toolchain_deps.bzl @@ -106,8 +106,8 @@ def cc_toolchain_deps(): # C++17, manylinux_2_27, gcc-8 mirrored_http_archive( name = "sysroot_linux_aarch64_glibc_2_27", - sha256 = "df19ac8b1415d1470ae721a3375e6111acdcf10756fc327bf78a0897249374e3", - mirrored_tar_sha256 = "f05a60f4cc1eaf6fa607cd59690218c3cbb8eddfab32e168e89c3cb4aaddbb22", + sha256 = "6168a4d40caed088a59c7c9f3c99900e1795002e8aaca43522a6391c05988c2e", + mirrored_tar_sha256 = "854dce12fa5a92001ac728f6b7a2101b3cdb0356b83e987022aaa7b2a4b77202", urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/sysroot_aarch64_ubuntu18_gcc8.4-0.2.0.tar.xz"), build_file = Label("//cc/config/sysroot_aarch64_linux_glibc_2_27:sysroot_aarch64_ubuntu18.BUILD"), strip_prefix = "sysroot_aarch64_ubuntu18_gcc8.4-0.2.0", From 3b28c063e02101719563b510dae353d21a3a4d52 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 06:23:51 +0000 Subject: [PATCH 08/20] Update sysroots --- .../BUILD.bazel => create-sysroot.sh} | 0 .../BUILD.bazel | 0 .../Dockerfile | 0 .../create-sysroot.sh | 8 +++++ .../sysroot_aarch64_ubuntu18.BUILD | 0 .../BUILD.bazel | 0 .../Dockerfile | 0 .../create-sysroot.sh | 31 +++++++++++++++++++ .../sysroot_x86_64_ubuntu18.BUILD | 0 cc/deps/cc_toolchain_deps.bzl | 13 ++++---- 10 files changed, 46 insertions(+), 6 deletions(-) rename cc/config/{sysroot_aarch64_linux_glibc_2_27/BUILD.bazel => create-sysroot.sh} (100%) rename cc/config/{sysroot_x86_64_linux_glibc_2_27 => sysroot_aarch64_ubuntu18_gcc8.4}/BUILD.bazel (100%) rename cc/config/{sysroot_aarch64_linux_glibc_2_27 => sysroot_aarch64_ubuntu18_gcc8.4}/Dockerfile (100%) rename cc/config/{sysroot_aarch64_linux_glibc_2_27 => sysroot_aarch64_ubuntu18_gcc8.4}/create-sysroot.sh (86%) rename cc/config/{sysroot_aarch64_linux_glibc_2_27 => sysroot_aarch64_ubuntu18_gcc8.4}/sysroot_aarch64_ubuntu18.BUILD (100%) create mode 100644 cc/config/sysroot_x86_64_ubuntu18_gcc8.4/BUILD.bazel rename cc/config/{sysroot_x86_64_linux_glibc_2_27 => sysroot_x86_64_ubuntu18_gcc8.4}/Dockerfile (100%) create mode 100644 cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh rename cc/config/{sysroot_x86_64_linux_glibc_2_27 => sysroot_x86_64_ubuntu18_gcc8.4}/sysroot_x86_64_ubuntu18.BUILD (100%) diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/BUILD.bazel b/cc/config/create-sysroot.sh similarity index 100% rename from cc/config/sysroot_aarch64_linux_glibc_2_27/BUILD.bazel rename to cc/config/create-sysroot.sh diff --git a/cc/config/sysroot_x86_64_linux_glibc_2_27/BUILD.bazel b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/BUILD.bazel similarity index 100% rename from cc/config/sysroot_x86_64_linux_glibc_2_27/BUILD.bazel rename to cc/config/sysroot_aarch64_ubuntu18_gcc8.4/BUILD.bazel diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/Dockerfile similarity index 100% rename from cc/config/sysroot_aarch64_linux_glibc_2_27/Dockerfile rename to cc/config/sysroot_aarch64_ubuntu18_gcc8.4/Dockerfile diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/create-sysroot.sh b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh similarity index 86% rename from cc/config/sysroot_aarch64_linux_glibc_2_27/create-sysroot.sh rename to cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh index 59de59ce..b59a343f 100644 --- a/cc/config/sysroot_aarch64_linux_glibc_2_27/create-sysroot.sh +++ b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh @@ -1,5 +1,13 @@ #!/bin/bash +if [ "$#" -ne 2 ]; then + echo "Usage (subscript): $0 " + exit 1 +fi + +SYSROOT="$1" +VERSION="$2" + ARCH_NAME=sysroot_aarch64_ubuntu18_gcc8.4-0.2.0 # Remove old files and image diff --git a/cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD similarity index 100% rename from cc/config/sysroot_aarch64_linux_glibc_2_27/sysroot_aarch64_ubuntu18.BUILD rename to cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/BUILD.bazel b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/cc/config/sysroot_x86_64_linux_glibc_2_27/Dockerfile b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile similarity index 100% rename from cc/config/sysroot_x86_64_linux_glibc_2_27/Dockerfile rename to cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh new file mode 100644 index 00000000..0c730097 --- /dev/null +++ b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +if [ "$#" -ne 3 ]; then + echo "Usage (subscript): $0 " + exit 1 +fi + +SYSROOT="$1" +VERSION="$2" +CONTAINER="$3" +ARCH_NAME=$SYSROOT-$VERSION + +# Create docker image from Dockerfile +docker build -t $ARCH_NAME:latest . + +# Run docker image +docker run -d --name $CONTAINER $ARCH_NAME:latest bash -c "sleep 20" + +mkdir /tmp/$ARCH_NAME + +# Copy needed directories from Docker image +docker cp $CONTAINER:/lib /tmp/$ARCH_NAME/ +docker cp $CONTAINER:/usr /tmp/$ARCH_NAME/ +# Remove not used directories +rm -rf /tmp/$ARCH_NAME/lib/cpp +rm -rf /tmp/$ARCH_NAME/usr/bin +rm -rf /tmp/$ARCH_NAME/usr/games +rm -rf /tmp/$ARCH_NAME/usr/sbin +rm -rf /tmp/$ARCH_NAME/usr/share +rm -rf /tmp/$ARCH_NAME/usr/src + diff --git a/cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD similarity index 100% rename from cc/config/sysroot_x86_64_linux_glibc_2_27/sysroot_x86_64_ubuntu18.BUILD rename to cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD diff --git a/cc/deps/cc_toolchain_deps.bzl b/cc/deps/cc_toolchain_deps.bzl index bb0a3a33..78b3f3de 100644 --- a/cc/deps/cc_toolchain_deps.bzl +++ b/cc/deps/cc_toolchain_deps.bzl @@ -59,13 +59,14 @@ def cc_toolchain_deps(): if "sysroot_linux_x86_64_glibc_2_27" not in native.existing_rules(): # C++17, manylinux_2_27, gcc-8 + # TODO: !!!!!!!!!!!!!!! Add C++ patch !!!!!!!!!!!!!!!! mirrored_http_archive( name = "sysroot_linux_x86_64_glibc_2_27", - sha256 = "7c54a97a8f62d467d493f2942f9063d5ea3d6ec012fcecf4ba62ababe1889be5", - mirrored_tar_sha256 = "4d48dff78a648cf6a210f409e9b807a0bd212eda34708c9374251d8c8b6b4808", - urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/ubuntu18_x86_64_sysroot_gcc8_patched-0.1.1.tar.xz"), - build_file = Label("//cc/config/sysroot_x86_64_linux_glibc_2_27:sysroot_x86_64_ubuntu18.BUILD"), - strip_prefix = "ubuntu18_x86_64_sysroot_gcc8_patched-0.1.1", + sha256 = "33d65928d3c0a79702b10ceac2a172666aa8997822abc3d10f1ce0ba1fe7c1bf", + mirrored_tar_sha256 = "18496de0f6eb0dd4cfbab4b618c0a50e8082989779acd048b68e760bf4d5b7d5", + urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/sysroot_x86_64_ubuntu18_gcc8.4-0.2.0.tar.xz"), + build_file = Label("//cc/config/sysroot_x86_64_ubuntu18_gcc8.4:sysroot_x86_64_ubuntu18.BUILD"), + strip_prefix = "sysroot_x86_64_ubuntu18_gcc8.4-0.2.0", ) if "sysroot_linux_x86_64_glibc_2_31" not in native.existing_rules(): @@ -109,7 +110,7 @@ def cc_toolchain_deps(): sha256 = "6168a4d40caed088a59c7c9f3c99900e1795002e8aaca43522a6391c05988c2e", mirrored_tar_sha256 = "854dce12fa5a92001ac728f6b7a2101b3cdb0356b83e987022aaa7b2a4b77202", urls = tf_mirror_urls("https://storage.googleapis.com/ml-sysroot-testing/sysroot_aarch64_ubuntu18_gcc8.4-0.2.0.tar.xz"), - build_file = Label("//cc/config/sysroot_aarch64_linux_glibc_2_27:sysroot_aarch64_ubuntu18.BUILD"), + build_file = Label("//cc/config/sysroot_aarch64_ubuntu18_gcc8.4:sysroot_aarch64_ubuntu18.BUILD"), strip_prefix = "sysroot_aarch64_ubuntu18_gcc8.4-0.2.0", ) From 791761f6317fa86cf2c65d9e133169be33bd766f Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 06:31:16 +0000 Subject: [PATCH 09/20] Create bash scripts to create image --- cc/config/create-sysroot.sh | 0 .../create-sysroot.sh | 52 +++++++++++-------- .../sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile | 40 +++++++++++--- .../create-sysroot.sh | 26 +++++++--- 4 files changed, 82 insertions(+), 36 deletions(-) delete mode 100644 cc/config/create-sysroot.sh mode change 100644 => 100755 cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh diff --git a/cc/config/create-sysroot.sh b/cc/config/create-sysroot.sh deleted file mode 100644 index e69de29b..00000000 diff --git a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh index b59a343f..5ffca4c5 100644 --- a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh +++ b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh @@ -1,38 +1,44 @@ #!/bin/bash -if [ "$#" -ne 2 ]; then - echo "Usage (subscript): $0 " - exit 1 +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " fi -SYSROOT="$1" -VERSION="$2" +SYSROOT="$(basename "$(pwd)")" +CONTAINER=$(echo "$SYSROOT" | sed -E 's/.*ubuntu([0-9]+).*/u\1sysroot/') -ARCH_NAME=sysroot_aarch64_ubuntu18_gcc8.4-0.2.0 +echo "Please enter sysroot version (default: 0.1.0):" +read VERSION +if [ -z "$VERSION" ]; then + VERSION="0.1.0" +fi + +ARCH_NAME=$SYSROOT-$VERSION # Remove old files and image -rm -f ./$ARCH_NAME/$ARCH_NAME.tar.xz -rm -rf ./$ARCH_NAME -docker rm -f u18sysroot +rm -rf /tmp/$ARCH_NAME +docker rm -f $CONTAINER docker rmi -f $ARCH_NAME:latest # Create docker image from Dockerfile docker build -t $ARCH_NAME:latest . -mkdir ./$ARCH_NAME - # Run docker image -docker run -d --name u18sysroot $ARCH_NAME:latest bash -c "sleep 20" +docker run -d --name $CONTAINER $ARCH_NAME:latest bash -c "sleep 20" + +mkdir /tmp/$ARCH_NAME # Copy needed directories from Docker image -docker cp u18sysroot:/lib ./$ARCH_NAME/ -docker cp u18sysroot:/usr ./$ARCH_NAME/ -rm -rf ./$ARCH_NAME/lib/cpp -rm -rf ./$ARCH_NAME/usr/bin -rm -rf ./$ARCH_NAME/usr/games -rm -rf ./$ARCH_NAME/usr/sbin -rm -rf ./$ARCH_NAME/usr/share -rm -rf ./$ARCH_NAME/usr/src - -echo "Creating $ARCH_NAME.tar.xz archive..." -tar cf - $ARCH_NAME | xz -T8 -c > $ARCH_NAME.tar.xz +docker cp $CONTAINER:/lib /tmp/$ARCH_NAME/ +docker cp $CONTAINER:/usr /tmp/$ARCH_NAME/ +# Remove not used directories +rm -rf /tmp/$ARCH_NAME/lib/cpp +rm -rf /tmp/$ARCH_NAME/usr/bin +rm -rf /tmp/$ARCH_NAME/usr/games +rm -rf /tmp/$ARCH_NAME/usr/sbin +rm -rf /tmp/$ARCH_NAME/usr/share +rm -rf /tmp/$ARCH_NAME/usr/src + +echo "Creating /tmp/$ARCH_NAME.tar.xz archive..." +XZ_OPT="-T8" +tar -cJf /tmp/$ARCH_NAME.tar.xz -C /tmp/ $ARCH_NAME \ No newline at end of file diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile index 4afb94b8..40712e8c 100644 --- a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile +++ b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile @@ -21,6 +21,7 @@ RUN rm -rf /usr/include/c++/7.5.0 RUN rm -rf /usr/include/c++/7 RUN rm -rf /usr/include/x86_64-linux-gnu/c++/7.5.0 RUN rm -rf /usr/include/x86_64-linux-gnu/c++/7 +RUN rm -rf /usr/lib/gcc/x86_64-linux-gnu/7 # Fix invalid links WORKDIR /lib64 @@ -32,14 +33,39 @@ RUN ln -sfn ../lib/x86_64-linux-gnu/ld-2.27.so ./ld-linux-x86-64.so.2 WORKDIR /usr/lib/x86_64-linux-gnu -# Fix /usr/lib/x86_64-linux-gnu/libdl.so -> /lib/x86_64-linux-gnu/libdl.so.2 -# cd usr/lib/x86_64-linux-gnu -# sudo ln -sfn ../../../lib/x86_64-linux-gnu/libdl.so.2 ./libdl.so +# /usr/lib/x86_64-linux-gnu/libutil.so -> /lib/x86_64-linux-gnu/libutil.so.1 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libutil.so.1 ./libutil.so +# /usr/lib/x86_64-linux-gnu/libnsl.so -> /lib/x86_64-linux-gnu/libnsl.so.1 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libnsl.so.1 ./libnsl.so +# /usr/lib/x86_64-linux-gnu/libcidn.so -> /lib/x86_64-linux-gnu/libcidn.so.1 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libcidn.so.1 ./libcidn.so +# /usr/lib/x86_64-linux-gnu/libnss_dns.so -> /lib/x86_64-linux-gnu/libnss_dns.so.2 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libnss_dns.so.2 ./libnss_dns.so +# /usr/lib/x86_64-linux-gnu/libnss_hesiod.so -> /lib/x86_64-linux-gnu/libnss_hesiod.so.2 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libnss_hesiod.so.2 ./libnss_hesiod.so +# /usr/lib/x86_64-linux-gnu/librt.so -> /lib/x86_64-linux-gnu/librt.so.1 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/librt.so.1 ./librt.so +# /usr/lib/x86_64-linux-gnu/libnss_nisplus.so -> /lib/x86_64-linux-gnu/libnss_nisplus.so.2 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libnss_nisplus.so.2 ./libnss_nisplus.so +# /usr/lib/x86_64-linux-gnu/libcrypt.so -> /lib/x86_64-linux-gnu/libcrypt.so.1 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libcrypt.so.1 ./libcrypt.so +# /usr/lib/x86_64-linux-gnu/libnss_compat.so -> /lib/x86_64-linux-gnu/libnss_compat.so.2 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libnss_compat.so.2 ./libnss_compat.so +# /usr/lib/x86_64-linux-gnu/libBrokenLocale.so -> /lib/x86_64-linux-gnu/libBrokenLocale.so.1 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libBrokenLocale.so.1 ./libBrokenLocale.so +# /usr/lib/x86_64-linux-gnu/libnss_files.so -> /lib/x86_64-linux-gnu/libnss_files.so.2 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libnss_files.so.2 ./libnss_files.so +# /usr/lib/x86_64-linux-gnu/libresolv.so -> /lib/x86_64-linux-gnu/libresolv.so.2 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libresolv.so.2 ./libresolv.so +# /usr/lib/x86_64-linux-gnu/libnss_nis.so -> /lib/x86_64-linux-gnu/libnss_nis.so.2 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libnss_nis.so.2 ./libnss_nis.so +# /usr/lib/x86_64-linux-gnu/libthread_db.so -> /lib/x86_64-linux-gnu/libthread_db.so.1 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libthread_db.so.1 ./libthread_db.so +# /usr/lib/x86_64-linux-gnu/libanl.so -> /lib/x86_64-linux-gnu/libanl.so.1 +RUN ln -sfn ../../../lib/x86_64-linux-gnu/libanl.so.1 ./libanl.so +# /usr/lib/x86_64-linux-gnu/libdl.so -> /lib/x86_64-linux-gnu/libdl.so.2 RUN ln -sfn ../../../lib/x86_64-linux-gnu/libdl.so.2 ./libdl.so - -# Fix /usr/lib/x86_64-linux-gnu/libmvec.so -> /lib/x86_64-linux-gnu/libmvec.so.1 -# cd usr/lib/x86_64-linux-gnu/ -# sudo ln -sfn ../../../lib/x86_64-linux-gnu/libmvec.so.1 ./libmvec.so +# /usr/lib/x86_64-linux-gnu/libmvec.so -> /lib/x86_64-linux-gnu/libmvec.so.1 RUN ln -sfn ../../../lib/x86_64-linux-gnu/libmvec.so.1 ./libmvec.so WORKDIR /root diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh old mode 100644 new mode 100755 index 0c730097..d9a871db --- a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh +++ b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh @@ -1,15 +1,25 @@ #!/bin/bash -if [ "$#" -ne 3 ]; then - echo "Usage (subscript): $0 " - exit 1 +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " +fi + +SYSROOT="$(basename "$(pwd)")" +CONTAINER=$(echo "$SYSROOT" | sed -E 's/.*ubuntu([0-9]+).*/u\1sysroot/') + +echo "Please enter sysroot version (default: 0.1.0):" +read VERSION +if [ -z "$VERSION" ]; then + VERSION="0.1.0" fi -SYSROOT="$1" -VERSION="$2" -CONTAINER="$3" ARCH_NAME=$SYSROOT-$VERSION +# Remove old files and image +rm -rf /tmp/$ARCH_NAME +docker rm -f $CONTAINER +docker rmi -f $ARCH_NAME:latest + # Create docker image from Dockerfile docker build -t $ARCH_NAME:latest . @@ -20,6 +30,7 @@ mkdir /tmp/$ARCH_NAME # Copy needed directories from Docker image docker cp $CONTAINER:/lib /tmp/$ARCH_NAME/ +docker cp $CONTAINER:/lib64 /tmp/$ARCH_NAME/ docker cp $CONTAINER:/usr /tmp/$ARCH_NAME/ # Remove not used directories rm -rf /tmp/$ARCH_NAME/lib/cpp @@ -29,3 +40,6 @@ rm -rf /tmp/$ARCH_NAME/usr/sbin rm -rf /tmp/$ARCH_NAME/usr/share rm -rf /tmp/$ARCH_NAME/usr/src +echo "Creating /tmp/$ARCH_NAME.tar.xz archive..." +XZ_OPT="-T8" +tar -cJf /tmp/$ARCH_NAME.tar.xz -C /tmp/ $ARCH_NAME \ No newline at end of file From c2d7f5fc9fc18fe0d5b045fdccf6c547b2354604 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 17:11:30 +0000 Subject: [PATCH 10/20] Fix invalid repo name for BZLMOD --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index cccbd169..1144af20 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -78,7 +78,7 @@ use_repo( "llvm_linux_x86_64", "sysroot_darwin_aarch64", "sysroot_linux_aarch64", - "sysroot_aarch64_linux_glibc_2_27", + "sysroot_linux_aarch64_glibc_2_27", "sysroot_linux_aarch64_glibc_2_31", "sysroot_linux_x86_64", "sysroot_linux_x86_64_glibc_2_27", From e229fc60245681baf8503cc35639b5fd47ec8c46 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 18:03:28 +0000 Subject: [PATCH 11/20] Remove deprecated comments --- WORKSPACE | 9 --------- 1 file changed, 9 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 00f516a7..8c9ea742 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -59,15 +59,6 @@ http_archive( urls = ["https://github.com/protocolbuffers/protobuf/archive/refs/tags/v5.28.3.zip"], ) -#http_archive( -# name = "com_google_protobuf", -# sha256 = "6e09bbc950ba60c3a7b30280210cd285af8d7d8ed5e0a6ed101c72aff22e8d88", -# strip_prefix = "protobuf-6.31.1", -# urls = ["https://github.com/protocolbuffers/protobuf/archive/refs/tags/v6.31.1.zip"], -# patches = ["//third_party:protobuf.patch"], -# patch_args = ["-p1"], -#) - load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") # Cross compilation error with older version. Details: https://github.com/bulletphysics/bullet3/issues/4607 From 5e957a9173bab54579b17438c23f916c66ab0de2 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 18:05:24 +0000 Subject: [PATCH 12/20] Remove unused openmp_lib --- .../sysroot_aarch64_ubuntu18.BUILD | 6 ------ .../sysroot_x86_64_ubuntu18.BUILD | 6 ------ 2 files changed, 12 deletions(-) diff --git a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD index 8da05170..f9dfd647 100644 --- a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD +++ b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD @@ -190,12 +190,6 @@ cc_toolchain_import( visibility = ["//visibility:public"], ) -cc_import( - name = "openmp_lib", - shared_library = "usr/lib/aarch64-linux-gnu/libomp.so", - visibility = ["//visibility:public"], -) - # This is a group of all the system libraries we need. The actual essential libraries is split # out to fix link ordering problems that cause false undefined symbol positives. cc_toolchain_import( diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD index e8d12779..65f0f84c 100644 --- a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD +++ b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD @@ -198,12 +198,6 @@ cc_toolchain_import( visibility = ["//visibility:public"], ) -cc_import( - name = "openmp_lib", - shared_library = "usr/lib/x86_64-linux-gnu/libomp.so", - visibility = ["//visibility:public"], -) - # This is a group of all the system libraries we need. The actual essential libraries is split # out to fix link ordering problems that cause false undefined symbol positives. cc_toolchain_import( From e279fdb08d7a2378afb5652ee183c7e8214e4f96 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 18:13:05 +0000 Subject: [PATCH 13/20] Add License headers --- .../Dockerfile | 14 ++++++++++ .../create-sysroot.sh | 15 +++++++++++ .../sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile | 26 ++++++++++--------- .../create-sysroot.sh | 15 +++++++++++ .../sysroot_x86_64_ubuntu18.BUILD | 1 - 5 files changed, 58 insertions(+), 13 deletions(-) diff --git a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/Dockerfile b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/Dockerfile index 6bbaea0d..bbaae395 100644 --- a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/Dockerfile +++ b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== FROM arm64v8/ubuntu:bionic diff --git a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh index 5ffca4c5..654c7821 100644 --- a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh +++ b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh @@ -1,5 +1,20 @@ #!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + if [ "$#" -ne 1 ]; then echo "Usage: $0 " fi diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile index 40712e8c..7ad57e42 100644 --- a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile +++ b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/Dockerfile @@ -1,13 +1,17 @@ -# Create docker image from Dockerfile -# docker build -t sysroot_ubuntu18_x86_64:latest -f . - -# Run docker image -# docker run -it sysroot_ubuntu18_x86_64 - -# Copy needed directories from Docker image -# docker cp :/usr . -# docker cp :/lib . -# docker cp :/lib64 . +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== FROM ubuntu:18.04 @@ -69,5 +73,3 @@ RUN ln -sfn ../../../lib/x86_64-linux-gnu/libdl.so.2 ./libdl.so RUN ln -sfn ../../../lib/x86_64-linux-gnu/libmvec.so.1 ./libmvec.so WORKDIR /root - -# tar cf - sysroot_x86_64_ubuntu18_gcc8.4-X.Y.Z | xz -T8 -c > sysroot_x86_64_ubuntu18_gcc8.4-X.Y.Z.tar.xz diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh index d9a871db..0cd29da8 100755 --- a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh +++ b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh @@ -1,5 +1,20 @@ #!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + if [ "$#" -ne 1 ]; then echo "Usage: $0 " fi diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD index 65f0f84c..b9050204 100644 --- a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD +++ b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/sysroot_x86_64_ubuntu18.BUILD @@ -1,5 +1,4 @@ # Copyright 2025 Google LLC -# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From b026b10b45e603bf38fb50ed1e711a148444548e Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 18:18:19 +0000 Subject: [PATCH 14/20] Remove enable_openmp flag --- .bazelrc | 1 - common/BUILD | 15 --------------- 2 files changed, 16 deletions(-) diff --git a/.bazelrc b/.bazelrc index 22501649..2dabc961 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,7 +4,6 @@ common:bzlmod --enable_bzlmod --noenable_workspace common --incompatible_enable_cc_toolchain_resolution # Only for bazel 6.5: C++ rules use platforms to select toolchains when you set this build --toolchain_resolution_debug=.* build --cxxopt=-std=c++17 -#build --@rules_ml_toolchain//common:enable_openmp=True common:clang_local --noincompatible_enable_cc_toolchain_resolution build:clang_local --@rules_ml_toolchain//common:enable_hermetic_cc=False diff --git a/common/BUILD b/common/BUILD index 2c8da6a3..62393ea2 100644 --- a/common/BUILD +++ b/common/BUILD @@ -59,21 +59,6 @@ config_setting( }, ) -####################################################### -# Enable OpenMP - -bool_flag( - name = "enable_openmp", - build_setting_default = False, -) - -config_setting( - name = "is_openmp_enabled", - flag_values = { - ":enable_openmp": "True", - }, -) - ####################################################### # Enable CUDA support flags # From 3e297451c76bd0ba6aebd3d20d4c8f86ee2f157e Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 18:23:16 +0000 Subject: [PATCH 15/20] Remove cc_toolchain_import_openmp_feature --- cc/features/openmp_feature.bzl | 122 ----------------------- cc/impls/linux_x86_64_linux_x86_64/BUILD | 1 - 2 files changed, 123 deletions(-) delete mode 100644 cc/features/openmp_feature.bzl diff --git a/cc/features/openmp_feature.bzl b/cc/features/openmp_feature.bzl deleted file mode 100644 index 507ad83d..00000000 --- a/cc/features/openmp_feature.bzl +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -load( - "@rules_cc//cc:action_names.bzl", - "ACTION_NAMES", - "ACTION_NAME_GROUPS", - "ALL_CC_COMPILE_ACTION_NAMES", - "ALL_CPP_COMPILE_ACTION_NAMES", - "CC_LINK_EXECUTABLE_ACTION_NAMES", - "DYNAMIC_LIBRARY_LINK_ACTION_NAMES", -) -load( - "@rules_cc//cc:cc_toolchain_config_lib.bzl", - "FeatureInfo", - "env_entry", - "env_set", - "feature", - "flag_group", - "flag_set", - _feature = "feature", -) -load( - "@rules_ml_toolchain//third_party/rules_cc_toolchain/features:cc_toolchain_import.bzl", - "CcToolchainImportInfo", -) - -def _get_origin_path(lib_file): - relative_path = lib_file.short_path - print("_get_origin_path: relative_path = ", relative_path) - - repo_name = lib_file.owner.workspace_name - - path_in_repo = lib_file.path.removesuffix(lib_file.basename).removesuffix(repo_name + '/') - - origin_prefix = "$ORIGIN/../../../../../../" - print("_get_origin_path: Result: ", origin_prefix + "external/" + repo_name + "/" + path_in_repo) - return origin_prefix + "external/" + repo_name + "/" + path_in_repo - -def _import_openmp_feature_impl(ctx): - - print("_import_openmp_feature_impl: ==============================================", dir(ctx)) - print("_import_openmp_feature_impl: bin_dir = ", ctx.bin_dir.path) - print("_import_openmp_feature_impl: build_file_path = ", ctx.build_file_path) - print("_import_openmp_feature_impl: genfiles_dir = ", ctx.genfiles_dir) - print("_import_openmp_feature_impl: runfiles = ", dir(ctx.runfiles)) - #print("_import_openmp_feature_impl: toolchain_import.name =", ctx.attr.toolchain_import.name) - #print("_import_openmp_feature_impl: toolchain_import.package =", ctx.attr.toolchain_import.package) - #print("_import_openmp_feature_impl: toolchain_import.workspace_name =", ctx.attr.toolchain_import.workspace_name) - print("_import_openmp_feature_impl: dir(...) =", dir(ctx.attr.toolchain_import)) - print("_import_openmp_feature_impl: workspace_root =", ctx.attr.toolchain_import.label.workspace_root) - print("_import_openmp_feature_impl: workspace_name =", ctx.attr.toolchain_import.label.workspace_name) - print("_import_openmp_feature_impl: package =", ctx.attr.toolchain_import.label.package) - print("_import_openmp_feature_impl: data_runfiles =", dir(ctx.attr.toolchain_import.data_runfiles)) - - runtime_paths = {} - for file in ctx.attr.toolchain_import.files.to_list(): - print("_import_openmp_feature_impl: file.basename =", file.basename) - print("_import_openmp_feature_impl: file.dirname =", file.dirname) - print("_import_openmp_feature_impl: file.extension =", file.extension) - print("_import_openmp_feature_impl: file.is_source =", file.is_source) - print("_import_openmp_feature_impl: file.path =", file.path) - print("_import_openmp_feature_impl: file.root =", dir(file.root)) - print("_import_openmp_feature_impl: file.short_path =", file.short_path) - print("_import_openmp_feature_impl: dir(file) =", dir(file)) - #print("_import_openmp_feature_impl: file.tree_relative_path =", file.tree_relative_path) - #print("_import_openmp_feature_impl: origin_path =", _get_origin_path(file)) - - - if file.path not in runtime_paths: - runtime_paths[file.dirname] = file - - toolchain_import_info = ctx.attr.toolchain_import[CcToolchainImportInfo] - - linker_runtime_path_flags = depset([ - "-Wl,-rpath," + _get_origin_path(runtime_paths[rpath]) - for rpath in runtime_paths - ]).to_list() - - flag_sets = [] - if linker_runtime_path_flags: - flag_sets.append(flag_set( - actions = CC_LINK_EXECUTABLE_ACTION_NAMES, - flag_groups = [ - flag_group( - flags = linker_runtime_path_flags, - ), - ], - )) - - library_feature = _feature( - name = ctx.label.name, - enabled = ctx.attr.enabled, - flag_sets = flag_sets, - implies = ctx.attr.implies, - provides = ctx.attr.provides, - ) - return [library_feature, ctx.attr.toolchain_import[DefaultInfo]] - -cc_toolchain_import_openmp_feature = rule( - _import_openmp_feature_impl, - attrs = { - "enabled": attr.bool(default = False), - "provides": attr.string_list(), - "requires": attr.string_list(), - "implies": attr.string_list(), - "toolchain_import": attr.label(providers = [CcToolchainImportInfo]), - }, - provides = [FeatureInfo, DefaultInfo], -) diff --git a/cc/impls/linux_x86_64_linux_x86_64/BUILD b/cc/impls/linux_x86_64_linux_x86_64/BUILD index 31e1dc7f..7761a31b 100644 --- a/cc/impls/linux_x86_64_linux_x86_64/BUILD +++ b/cc/impls/linux_x86_64_linux_x86_64/BUILD @@ -20,7 +20,6 @@ load("//third_party/rules_cc_toolchain:toolchain_config.bzl", "cc_toolchain_config") load("//third_party/rules_cc_toolchain/features:cc_toolchain_import.bzl", "cc_toolchain_import") load("//third_party/rules_cc_toolchain/features:features.bzl", "cc_toolchain_import_feature") -load("//cc/features:openmp_feature.bzl", "cc_toolchain_import_openmp_feature") package( default_visibility = [ From d0de5ee5b8afe5a95dfffb79a84e9c36ffc9fec3 Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 22:27:57 +0000 Subject: [PATCH 16/20] Remove not used protobuf path --- third_party/protobuf.patch | 201 ------------------------------------- 1 file changed, 201 deletions(-) delete mode 100644 third_party/protobuf.patch diff --git a/third_party/protobuf.patch b/third_party/protobuf.patch deleted file mode 100644 index e54be199..00000000 --- a/third_party/protobuf.patch +++ /dev/null @@ -1,201 +0,0 @@ -diff --git a/BUILD.bazel b/BUILD.bazel ---- a/BUILD.bazel -+++ b/BUILD.bazel -@@ -555,7 +555,8 @@ proto_lang_toolchain( - "//:cpp_features_proto", - "//:descriptor_proto", - ], -- command_line = "--cpp_out=$(OUT)", -+ command_line = "--cpp_out=dllexport_decl=PROTOBUF_EXPORT:$(OUT)", -+# command_line = "--cpp_out=$(OUT)", - runtime = "//src/google/protobuf", - visibility = ["//visibility:public"], - ) -diff --git a/build_defs/BUILD.bazel b/build_defs/BUILD.bazel ---- a/build_defs/BUILD.bazel -+++ b/build_defs/BUILD.bazel -@@ -1,6 +1,7 @@ - # Internal Starlark definitions for Protobuf. - - load("@bazel_skylib//lib:selects.bzl", "selects") -+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") - load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") - load("//bazel:cc_proto_library.bzl", starlark_cc_proto_library = "cc_proto_library") - load(":cc_proto_blacklist_test.bzl", "cc_proto_blacklist_test") -@@ -13,6 +14,20 @@ package( - ], - ) - -+bool_flag( -+ name = "use_dlls", -+ build_setting_default = False, -+ visibility = ["//visibility:public"], -+) -+ -+config_setting( -+ name = "config_use_dlls", -+ flag_values = { -+ ":use_dlls": "True", -+ }, -+ visibility = ["//visibility:public"], -+) -+ - create_compiler_config_setting( - name = "config_msvc_cl", - value = "msvc-cl", -diff --git a/python/dist/system_python.bzl b/python/dist/system_python.bzl ---- a/python/dist/system_python.bzl -+++ b/python/dist/system_python.bzl -@@ -73,11 +73,10 @@ load("@bazel_skylib//lib:selects.bzl", "selects") - load("@bazel_skylib//rules:common_settings.bzl", "string_flag") - load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") - --cc_library( -- name = "python_headers", -- hdrs = glob(["python/**/*.h"], allow_empty = True), -- includes = ["python"], -- visibility = ["//visibility:public"], -+alias( -+ name = "python_headers", -+ actual = "@rules_python//python/cc:current_py_cc_headers", -+ visibility = ["//visibility:public"], - ) - - string_flag( -@@ -219,7 +218,7 @@ def _system_python_impl(repository_ctx): - python3 = repository_ctx.which("python3") - python_version = _get_python_version(repository_ctx) - -- if path and python_version[0] == "3": -+ if False: - _populate_package(repository_ctx, path, python3, python_version) - else: - # buildifier: disable=print -diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py ---- a/python/google/protobuf/__init__.py -+++ b/python/google/protobuf/__init__.py -@@ -8,3 +8,9 @@ - # Copyright 2007 Google Inc. All Rights Reserved. - - __version__ = '6.31.1' -+ -+if __name__ != '__main__': -+ try: -+ __import__('pkg_resources').declare_namespace(__name__) -+ except ImportError: -+ __path__ = __import__('pkgutil').extend_path(__path__, __name__) -diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel ---- a/src/google/protobuf/BUILD.bazel -+++ b/src/google/protobuf/BUILD.bazel -@@ -525,6 +525,13 @@ cc_library( - "serial_arena.h", - "thread_safe_arena.h", - ], -+ local_defines = select({ -+ "//build_defs:config_use_dlls": [ -+ "PROTOBUF_USE_DLLS", -+ "LIBPROTOBUF_EXPORTS", -+ ], -+ "//conditions:default": [], -+ }), - strip_include_prefix = "/src", - visibility = [ - "//:__subpackages__", -@@ -657,7 +664,15 @@ cc_library( - "serial_arena.h", - "thread_safe_arena.h", - "wire_format_lite.h", -+ "port.h", - ], -+ local_defines = select({ -+ "//build_defs:config_use_dlls": [ -+ "PROTOBUF_USE_DLLS", -+ "LIBPROTOBUF_EXPORTS", -+ ], -+ "//conditions:default": [], -+ }), - copts = COPTS + select({ - "//build_defs:config_msvc": [], - "//conditions:default": [ -@@ -767,6 +782,13 @@ cc_library( - ], - hdrs = PROTOBUF_HEADERS, - copts = COPTS, -+ local_defines = select({ -+ "//build_defs:config_use_dlls": [ -+ "PROTOBUF_USE_DLLS", -+ "LIBPROTOBUF_EXPORTS", -+ ], -+ "//conditions:default": [], -+ }), - linkopts = LINK_OPTS, - strip_include_prefix = "/src", - visibility = [ -diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc ---- a/src/google/protobuf/arena.cc -+++ b/src/google/protobuf/arena.cc -@@ -547,7 +547,7 @@ ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() { - new internal::ThreadLocalStorage(); - return *thread_cache_->Get(); - } --#elif defined(PROTOBUF_USE_DLLS) && defined(_WIN32) -+#elif defined(_WIN32) - ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() { - static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache; - return thread_cache; -diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel ---- a/src/google/protobuf/io/BUILD.bazel -+++ b/src/google/protobuf/io/BUILD.bazel -@@ -22,6 +22,13 @@ cc_library( - "zero_copy_stream_impl.h", - "zero_copy_stream_impl_lite.h", - ], -+ local_defines = select({ -+ "//build_defs:config_use_dlls": [ -+ "PROTOBUF_USE_DLLS", -+ "LIBPROTOBUF_EXPORTS", -+ ], -+ "//conditions:default": [], -+ }), - copts = COPTS, - strip_include_prefix = "/src", - deps = [ -diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h ---- a/src/google/protobuf/message_lite.h -+++ b/src/google/protobuf/message_lite.h -@@ -282,7 +282,11 @@ template - using MessageTraits = decltype(MessageTraitsImpl::value); - - struct EnumTraitsImpl { -+#ifdef __CUDACC__ -+ struct Undefined{}; -+#else - struct Undefined; -+#endif - template - static Undefined value; - }; -diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc ---- a/src/google/protobuf/port_def.inc -+++ b/src/google/protobuf/port_def.inc -@@ -421,7 +421,7 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), - #endif - - // Lexan sets both MSV_VER and clang, so handle it with the clang path. --#if defined(_MSC_VER) && !defined(__clang__) -+#if defined(_MSC_VER) - // MSVC 17 currently seems to raise an error about constant-initialized pointers. - # if PROTOBUF_MSC_VER_MIN(1930) - # define PROTOBUF_CONSTINIT -diff --git a/src/google/protobuf/thread_safe_arena.h b/src/google/protobuf/thread_safe_arena.h ---- a/src/google/protobuf/thread_safe_arena.h -+++ b/src/google/protobuf/thread_safe_arena.h -@@ -249,7 +249,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { - // iOS does not support __thread keyword so we use a custom thread local - // storage class we implemented. - static ThreadCache& thread_cache(); --#elif defined(PROTOBUF_USE_DLLS) && defined(_WIN32) -+#elif defined(_WIN32) - // Thread local variables cannot be exposed through MSVC DLL interface but we - // can wrap them in static functions. - static ThreadCache& thread_cache(); From 0131d6b2fa0769d4b7da216c91fab11a55d9519c Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Fri, 31 Oct 2025 22:33:25 +0000 Subject: [PATCH 17/20] Remove unused openmp code --- cc/impls/linux_x86_64_linux_x86_64/BUILD | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/cc/impls/linux_x86_64_linux_x86_64/BUILD b/cc/impls/linux_x86_64_linux_x86_64/BUILD index 7761a31b..207e80c2 100644 --- a/cc/impls/linux_x86_64_linux_x86_64/BUILD +++ b/cc/impls/linux_x86_64_linux_x86_64/BUILD @@ -60,28 +60,6 @@ cc_toolchain_import_feature( toolchain_import = ":imports", ) -#cc_toolchain_import( -# name = "openmp", - # features = select({ - # "//common:is_openmp_enabled": [ - # "openmp", - # ], - # "//conditions:default": [], - # }), -# deps = select({ -# "//common:is_openmp_enabled": [ -# "@sysroot_linux_x86_64//:openmp", -# ], -# "//conditions:default": [], -# }), -#) - -#cc_toolchain_import_feature( -# name = "openmp_feature", -# enabled = True, -# toolchain_import = ":openmp", -#) - # buildifier: leave-alone filegroup( name = "all", From 08130743dcb97b629ab869dd3071785b83cb9acf Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Sat, 1 Nov 2025 05:34:30 +0000 Subject: [PATCH 18/20] Remove runtime_path and add comments --- .../sysroot_aarch64_ubuntu18.BUILD | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD index f9dfd647..55922402 100644 --- a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD +++ b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD @@ -30,6 +30,9 @@ sysroot_package( GCC_VERSION = 8 GLIBC_VERSION = "2.27" +# Details about C RunTime (CRT) objects: +# https://docs.oracle.com/cd/E88353_01/html/E37853/crt1.o-7.html +# https://dev.gentoo.org/~vapier/crt.txt CRT_OBJECTS = [ "crti", "crtn", @@ -88,7 +91,6 @@ cc_toolchain_import( "lib/aarch64-linux-gnu/libgcc_s.so.1", "usr/lib/gcc/aarch64-linux-gnu/{gcc_version}/libgcc_eh.a".format(gcc_version = GCC_VERSION), ], - runtime_path = "/usr/lib/aarch64-linux-gnu", shared_library = "usr/lib/gcc/aarch64-linux-gnu/{gcc_version}/libgcc_s.so".format(gcc_version = GCC_VERSION), static_library = "usr/lib/gcc/aarch64-linux-gnu/{gcc_version}/libgcc.a".format(gcc_version = GCC_VERSION), visibility = ["//visibility:public"], @@ -166,10 +168,6 @@ cc_toolchain_import( ], shared_library = "usr/lib/aarch64-linux-gnu/libc.so", static_library = "usr/lib/aarch64-linux-gnu/libc.a", - #target_compatible_with = select({ - # "@platforms//os:linux": ["@platforms//cpu:aarch64"], - # "//conditions:default": ["@platforms//:incompatible"], - #}), visibility = ["//visibility:public"], deps = [ ":gcc", From e380bb37f512885eb1f1ceb8d6c5503e503c394c Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Sat, 1 Nov 2025 06:01:45 +0000 Subject: [PATCH 19/20] Remove runtime_path and comments to scripts --- cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh | 5 ++++- cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh index 654c7821..3417c5fe 100644 --- a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh +++ b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/create-sysroot.sh @@ -36,11 +36,14 @@ docker rm -f $CONTAINER docker rmi -f $ARCH_NAME:latest # Create docker image from Dockerfile +echo -e "\nBuilding docker image $ARCH_NAME:latest" docker build -t $ARCH_NAME:latest . # Run docker image +echo -e "\nRunning docker image $ARCH_NAME:latest, container name $CONTAINER" docker run -d --name $CONTAINER $ARCH_NAME:latest bash -c "sleep 20" +echo -e "\nCopying files from docker container..." mkdir /tmp/$ARCH_NAME # Copy needed directories from Docker image @@ -54,6 +57,6 @@ rm -rf /tmp/$ARCH_NAME/usr/sbin rm -rf /tmp/$ARCH_NAME/usr/share rm -rf /tmp/$ARCH_NAME/usr/src -echo "Creating /tmp/$ARCH_NAME.tar.xz archive..." +echo -e "\nCreating /tmp/$ARCH_NAME.tar.xz archive..." XZ_OPT="-T8" tar -cJf /tmp/$ARCH_NAME.tar.xz -C /tmp/ $ARCH_NAME \ No newline at end of file diff --git a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh index 0cd29da8..2de0dc17 100755 --- a/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh +++ b/cc/config/sysroot_x86_64_ubuntu18_gcc8.4/create-sysroot.sh @@ -36,11 +36,14 @@ docker rm -f $CONTAINER docker rmi -f $ARCH_NAME:latest # Create docker image from Dockerfile +echo -e "\nBuilding docker image $ARCH_NAME:latest" docker build -t $ARCH_NAME:latest . # Run docker image +echo -e "\nRunning docker image $ARCH_NAME:latest, container name $CONTAINER" docker run -d --name $CONTAINER $ARCH_NAME:latest bash -c "sleep 20" +echo -e "\nCopying files from docker container..." mkdir /tmp/$ARCH_NAME # Copy needed directories from Docker image @@ -55,6 +58,6 @@ rm -rf /tmp/$ARCH_NAME/usr/sbin rm -rf /tmp/$ARCH_NAME/usr/share rm -rf /tmp/$ARCH_NAME/usr/src -echo "Creating /tmp/$ARCH_NAME.tar.xz archive..." +echo -e "\nCreating /tmp/$ARCH_NAME.tar.xz archive..." XZ_OPT="-T8" tar -cJf /tmp/$ARCH_NAME.tar.xz -C /tmp/ $ARCH_NAME \ No newline at end of file From d49b603869a6d1ee2d7ed0550da932ad62ae5baf Mon Sep 17 00:00:00 2001 From: "Yurii T." Date: Sat, 1 Nov 2025 06:26:56 +0000 Subject: [PATCH 20/20] Fix Linux aarch64 dynamic linker and libc --- .../sysroot_aarch64_ubuntu18.BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD index 55922402..b602d74a 100644 --- a/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD +++ b/cc/config/sysroot_aarch64_ubuntu18_gcc8.4/sysroot_aarch64_ubuntu18.BUILD @@ -118,6 +118,7 @@ cc_toolchain_import( cc_toolchain_import( name = "dynamic_linker", additional_libs = [ + "lib/ld-linux-aarch64.so.1", "lib/aarch64-linux-gnu/ld-linux-aarch64.so.1", "lib/aarch64-linux-gnu/ld-{glibc_version}.so".format(glibc_version = GLIBC_VERSION), ], @@ -164,6 +165,7 @@ cc_toolchain_import( name = "libc", additional_libs = [ "lib/aarch64-linux-gnu/libc.so.6", + "lib/aarch64-linux-gnu/libc-{glibc_version}.so".format(glibc_version = GLIBC_VERSION), "usr/lib/aarch64-linux-gnu/libc_nonshared.a", ], shared_library = "usr/lib/aarch64-linux-gnu/libc.so",