Introduce load-balanced channel for OpenTelemetry exporters

Add client-side load balancing to OTLP gRPC connections using ginepro.
When NL_OTEL_ENDPOINT is set, the telemetry system creates a load-balanced
channel shared across log, trace, and metric exporters. This enables better
distribution of telemetry traffic across multiple OTLP collector instances
and improves overall system resilience.

- Add ginepro dependency for gRPC load balancing
- Upgrade OpenTelemetry dependencies from 0.29 to 0.30
- Change init_tracing() to async to support channel initialization
- Add NL_OTEL_ENDPOINT environment variable for configuration
- Update all OTLP exporters to use shared load-balanced channel

# Conflicts:
#	Cargo.lock

Author: dkostyrev

Cargo.lock

@@ -9,6 +9,7 @@ bazel_dep(name = "platforms", version = "1.1.0")
 bazel_dep(name = "bazel_skylib", version = "1.9.0")
 bazel_dep(name = "rules_python", version = "2.0.0")
 bazel_dep(name = "rules_shell", version = "0.6.1")
+bazel_dep(name = "zstd", version = "1.5.7.bcr.1")
 
 # hermetic-llvm: zero-sysroot, fully hermetic LLVM cross-compilation toolchain.
 # Inside Nix, lre.bazelrc registers @local-remote-execution//generated-cc/...
@@ -107,6 +108,14 @@ crate.from_cargo(
     # selects against legacy @rules_rust//rust/platform:* labels.
     use_legacy_rules_rust_platforms = True,
 )
+crate.annotation(
+    crate = "zstd-sys",
+    gen_build_script = "off",
+    deps = ["@zstd"],
+)
+
+inject_repo(crate, "zstd")
+
 use_repo(crate, "crates")
 
 # NativeLink uses Local Remote Execution for Rust by default which automatically

MODULE.bazel

@@ -121,6 +121,7 @@
             else "${pkgs.llvmPackages_22.lld}/bin/ld.lld";
 
           linkerEnvVar = "CARGO_TARGET_${pkgs.lib.toUpper (pkgs.lib.replaceStrings ["-"] ["_"] targetArch)}_LINKER";
+          cflagsEnvVar = "CFLAGS_${pkgs.lib.toUpper (pkgs.lib.replaceStrings ["-"] ["_"] targetArch)}";
         in
           {
             inherit src;
@@ -154,6 +155,11 @@
             # FIXME(palfrey): Attempted workaround from https://github.com/llvm/llvm-project/issues/32849#issuecomment-2353071071 but doesn't work
             # CFLAGS = "-femit-all-decls";
             ${linkerEnvVar} = linkerPath;
+            # musl libc does not provide __memcpy_chk (a glibc _FORTIFY_SOURCE extension).
+            # C crates compiled for musl (e.g. zstd-sys) fail to link when the compiler
+            # emits calls to that symbol. Undefining _FORTIFY_SOURCE disables the
+            # fortified-function substitution for the musl target.
+            ${cflagsEnvVar} = "-U_FORTIFY_SOURCE";
           });
 
         # Additional target for external dependencies to simplify caching.

MODULE.bazel.lock

@@ -21,8 +21,8 @@ futures = { version = "0.3.31", default-features = false }
 humantime = { version = "2.3.0", default-features = false }
 lru = { version = "0.16.0", default-features = false }
 mock_instant = { version = "0.5.3", default-features = false }
-opentelemetry = { version = "0.29.1", default-features = false }
-opentelemetry-semantic-conventions = { version = "0.29.0", default-features = false, features = [
+opentelemetry = { version = "0.30.0", default-features = false }
+opentelemetry-semantic-conventions = { version = "0.30.0", default-features = false, features = [
   "default",
   "semconv_experimental",
 ] }

flake.nix

@@ -20,8 +20,8 @@ bytes = { version = "1.10.1", default-features = false }
 futures = { version = "0.3.31", default-features = false }
 http-body-util = { version = "0.1.3", default-features = false }
 hyper = { version = "1.6.0", default-features = false }
-opentelemetry = { version = "0.29.1", default-features = false }
-opentelemetry-semantic-conventions = { version = "0.29.0", default-features = false, features = [
+opentelemetry = { version = "0.30.0", default-features = false }
+opentelemetry-semantic-conventions = { version = "0.30.0", default-features = false, features = [
   "default",
   "semconv_experimental",
 ] }

nativelink-scheduler/Cargo.toml

@@ -70,7 +70,7 @@ mongodb = { version = "3", features = [
   "compat-3-0-0",
   "rustls-tls",
 ], default-features = false }
-opentelemetry = { version = "0.29.1", default-features = false }
+opentelemetry = { version = "0.30.0", default-features = false }
 parking_lot = { version = "0.12.3", features = [
   "arc_lock",
   "send_guard",

nativelink-service/Cargo.toml

@@ -54,6 +54,7 @@ rust_library(
         "@crates//:blake3",
         "@crates//:bytes",
         "@crates//:futures",
+        "@crates//:ginepro",
         "@crates//:hex",
         "@crates//:humantime",
         "@crates//:hyper-1.7.0",
@@ -83,6 +84,7 @@ rust_library(
         "@crates//:tracing",
         "@crates//:tracing-opentelemetry",
         "@crates//:tracing-subscriber",
+        "@crates//:url",
         "@crates//:uuid",
         "@crates//:walkdir",
         "@crates//:wincode",
@@ -136,6 +138,7 @@ rust_test_suite(
         "@crates//:mock_instant",
         "@crates//:opentelemetry",
         "@crates//:opentelemetry-http",
+        "@crates//:opentelemetry_sdk",
         "@crates//:parking_lot",
         "@crates//:pretty_assertions",
         "@crates//:prost",
@@ -164,10 +167,13 @@ rust_test(
     ],
     deps = [
         "@crates//:http-body-util",
+        "@crates//:opentelemetry-proto",
+        "@crates//:opentelemetry_sdk",
         "@crates//:pretty_assertions",
         "@crates//:rand",
         "@crates//:serde_json",
         "@crates//:tempfile",
+        "@crates//:tokio-stream",
         "@crates//:tracing-test",
     ],
 )

nativelink-store/Cargo.toml

@@ -27,21 +27,21 @@ hyper-util = { version = "0.1.11", default-features = false }
 libc = { version = "0.2.177", default-features = false }
 lru = { version = "0.16.0", default-features = false }
 mock_instant = { version = "0.5.3", default-features = false }
-opentelemetry = { version = "0.29.0", default-features = false }
-opentelemetry-appender-tracing = { version = "0.29.1", default-features = false }
-opentelemetry-http = { version = "0.29.0", default-features = false }
-opentelemetry-otlp = { version = "0.29.0", default-features = false, features = [
+opentelemetry = { version = "0.30.0", default-features = false }
+opentelemetry-appender-tracing = { version = "0.30.0", default-features = false }
+opentelemetry-http = { version = "0.30.0", default-features = false }
+opentelemetry-otlp = { version = "0.30.0", default-features = false, features = [
   "grpc-tonic",
   "logs",
   "metrics",
   "trace",
   "zstd-tonic",
 ] }
-opentelemetry-semantic-conventions = { version = "0.29.0", default-features = false, features = [
+opentelemetry-semantic-conventions = { version = "0.30.0", default-features = false, features = [
   "default",
   "semconv_experimental",
 ] }
-opentelemetry_sdk = { version = "0.29.0", default-features = false }
+opentelemetry_sdk = { version = "0.30.0", default-features = false }
 parking_lot = { version = "0.12.3", features = [
   "arc_lock",
   "send_guard",
@@ -77,7 +77,7 @@ tonic = { version = "0.13.0", features = [
 ], default-features = false }
 tower = { version = "0.5.2", default-features = false }
 tracing = { version = "0.1.41", default-features = false }
-tracing-opentelemetry = { version = "0.30.0", default-features = false, features = [
+tracing-opentelemetry = { version = "0.31.0", default-features = false, features = [
   "metrics",
 ] }
 tracing-subscriber = { version = "0.3.19", features = [
@@ -87,6 +87,8 @@ tracing-subscriber = { version = "0.3.19", features = [
 ], default-features = false }
 tracing-test = { version = "0.2.5", default-features = false, features = [] }
 
+ginepro = { version = "0.9.3", default-features = false }
+url = { version = "2.5.7", default-features = false }
 uuid = { version = "1.16.0", default-features = false, features = [
   "serde",
   "v4",
@@ -97,6 +99,13 @@ wincode = { version = "0.5.4", default-features = false, features = ["derive"] }
 
 [dev-dependencies]
 nativelink-macro = { path = "../nativelink-macro" }
+opentelemetry-proto = { version = "0.30.0", default-features = false, features = [
+  "gen-tonic",
+  "metrics",
+] }
+opentelemetry_sdk = { version = "0.30.0", default-features = false, features = [
+  "testing",
+] }
 
 axum = { version = "0.8.3", default-features = false }
 http-body-util = { version = "0.1.3", default-features = false }