Skip to content

Commit 994e62c

Browse files
malkiamalkia
committed
Added hang repro case for grpc hangs. There is one hang using async export, and another (one in ~3000 runs) due to grpc using absl primitives. Added ext api to modify grpc experiment values
1 parent 7ff504f commit 994e62c

18 files changed

+245
-721
lines changed

‎.bazelrc

+8-3
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ startup --host_jvm_args=-DBAZEL_TRACK_SOURCE_DIRECTORIES=1
1111
# For some reason, with clang-cl, it also tried to do on Windows, where dynamic linking just does not support it.
1212
build --dynamic_mode=off
1313

14-
common --remote_download_minimal
14+
common --remote_download_all
1515

1616
# Avoid using c:\windows\system32\bash.exe
1717
common:windows --shell_executable="c:\\program files\\git\\usr\\bin\\bash.exe"
@@ -142,8 +142,13 @@ startup --client_debug
142142

143143
common --flag_alias=dll=@otel_sdk//:with_dll
144144

145-
# disable GRPC experiments
146-
build --define=grpc_experiments_are_final=true
145+
# Compiling GRPC without using ABSL sync primitives, as this causes rare hang on Windows
146+
# x/hang/hang_callstack.png
147+
build --copt="-DGRPC_NO_ABSL_SYNC"
148+
build --host_copt="-DGRPC_NO_ABSL_SYNC"
149+
150+
# disable all GRPC experiments
151+
# build --define=grpc_experiments_are_final=true
147152

148153
## This is what my ../top.bazelrc contains (not in the repo as local to my machine)
149154
# build --disk_cache=f:/b/d

‎MODULE.bazel

+7
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ bazel_dep(name = "gazelle", version = "0.42.0")
2929
bazel_dep(name = "stardoc", version = "0.8.0")
3030
bazel_dep(name = "rules_android", version = "0.6.3")
3131
bazel_dep(name = "grpc", version = "1.71.0")
32+
single_version_override(
33+
module_name = "grpc",
34+
patch_strip = 1,
35+
patches = [
36+
"bazel/grpc_no_absl_sync.patch",
37+
],
38+
)
3239
bazel_dep(name = "nlohmann_json", version = "3.11.3.bcr.1")
3340
bazel_dep(name = "platforms", version = "0.0.11")
3441
bazel_dep(name = "rules_cc", version = "0.1.1")

‎bazel/grpc_no_absl_sync.patch

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
diff --git a/include/grpc/support/port_platform.h b/include/grpc/support/port_platform.h
2+
index 04a79ef3c8..6db7822131 100644
3+
--- a/include/grpc/support/port_platform.h
4+
+++ b/include/grpc/support/port_platform.h
5+
@@ -30,7 +30,7 @@
6+
* Defines GPR_ABSEIL_SYNC to use synchronization features from Abseil
7+
*/
8+
#ifndef GPR_ABSEIL_SYNC
9+
-#if defined(__APPLE__)
10+
+#if defined(__APPLE__) || defined(GRPC_NO_ABSL_SYNC)
11+
// This is disabled on Apple platforms because macos/grpc_basictests_c_cpp
12+
// fails with this. https://github.com/grpc/grpc/issues/23661
13+
#else

‎ext/include/opentelemetry/ext/grpc/grpc_config.h

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
#pragma once
5+
6+
#include <optional>
7+
#include <string>
8+
#include <cstddef>
9+
10+
#include "opentelemetry/version.h"
11+
12+
OPENTELEMETRY_BEGIN_NAMESPACE
13+
namespace ext
14+
{
15+
namespace grpc
16+
{
17+
#define OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_ENUM(_) \
18+
_(int32_t, client_channel_backup_poll_interval_ms, ClientChannelBackupPollIntervalMs) \
19+
_(bool, enable_fork_support, EnableForkSupport) \
20+
_(bool, abort_on_leaks, AbortOnLeaks) \
21+
_(bool, not_use_system_ssl_roots, NotUseSystemSslRoots) \
22+
_(bool, cpp_experimental_disable_reflection, CppExperimentalDisableReflection) \
23+
_(std::string, dns_resolver, DnsResolver) \
24+
_(std::string, verbosity, Verbosity) \
25+
_(std::string, poll_strategy, PollStrategy) \
26+
_(std::string, system_ssl_roots_dir, SystemSslRootsDir) \
27+
_(std::string, default_ssl_roots_file_path, DefaultSslRootsFilePath) \
28+
_(std::string, ssl_cipher_suites, SslCipherSuites) \
29+
_(std::string, experiments, Experiments) \
30+
_(std::string, trace, Trace)
31+
32+
struct Overrides {
33+
#define OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_DEF(type,name,_) std::optional<type> name;
34+
OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_ENUM(OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_DEF)
35+
#undef OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_DEF
36+
};
37+
38+
OPENTELEMETRY_EXPORT std::string OverridesToString();
39+
OPENTELEMETRY_EXPORT void ResetOverrides();
40+
OPENTELEMETRY_EXPORT void SetOverrides(const Overrides& o);
41+
OPENTELEMETRY_EXPORT Overrides GetOverrides();
42+
} // namespace grpc
43+
} // namespace ext
44+
OPENTELEMETRY_END_NAMESPACE

‎ext/src/grpc/BUILD

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Copyright The OpenTelemetry Authors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
load("//bazel:otel_cc.bzl", "otel_cc_library")
5+
6+
package(default_visibility = ["//visibility:public"])
7+
8+
otel_cc_library(
9+
name = "grpc_config",
10+
srcs = ["grpc_config.cc"],
11+
deps = [
12+
"//api",
13+
"//ext:headers",
14+
"@grpc//:config_vars",
15+
#"@grpc//:grpc++",
16+
],
17+
)

‎ext/src/grpc/grpc_config.cc

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#include "opentelemetry/ext/grpc/grpc_config.h"
2+
3+
// including from grpc's innards
4+
#include <src/core/config/config_vars.h>
5+
6+
OPENTELEMETRY_BEGIN_NAMESPACE
7+
namespace ext
8+
{
9+
namespace grpc
10+
{
11+
12+
std::string OverridesToString()
13+
{
14+
return grpc_core::ConfigVars::Get().ToString();
15+
}
16+
17+
void ResetOverrides()
18+
{
19+
grpc_core::ConfigVars::Reset();
20+
}
21+
22+
void SetOverrides(const Overrides& o)
23+
{
24+
grpc_core::ConfigVars::Overrides go{};
25+
26+
#define OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_COPY_TO(_,name,__) go.name = o.name;
27+
OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_ENUM(OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_COPY_TO)
28+
29+
grpc_core::ConfigVars::SetOverrides(go);
30+
}
31+
32+
OPENTELEMETRY_EXPORT Overrides GetOverrides()
33+
{
34+
const auto& configVars{ grpc_core::ConfigVars::Get() };
35+
Overrides o{};
36+
37+
#define OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_COPY_FROM(_,name,prop) o.name = configVars.prop();
38+
OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_ENUM(OPENTELEMETRY_EXT_GRPC_OVERRIDES_X_COPY_FROM)
39+
40+
return o;
41+
}
42+
43+
} // namespace grpc
44+
} // namespace ext
45+
OPENTELEMETRY_END_NAMESPACE

‎ext/test/grpc/BUILD

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Copyright The OpenTelemetry Authors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
load("//:dll_deps.bzl", "dll_deps")
5+
load("//bazel:otel_cc.bzl", "otel_cc_test")
6+
7+
otel_cc_test(
8+
name = "grpc_config_test",
9+
srcs = ["grpc_config_test.cc"],
10+
deps = dll_deps([
11+
"//ext/src/grpc:grpc_config",
12+
"@googletest//:gtest_main",
13+
"@grpc//:grpc++",
14+
]),
15+
env = {
16+
"GRPC_VERBOSITY": "debug",
17+
"GRPC_EXPERIMENTS": ",".join([
18+
"-event_engine_client",
19+
"-event_engine_listener",
20+
"-event_engine_dns,",
21+
]),
22+
}
23+
)
24+

‎ext/test/grpc/grpc_config_test.cc

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
#include <gtest/gtest.h>
5+
#include <gmock/gmock.h>
6+
#include <stdio.h>
7+
8+
#include <opentelemetry/ext/grpc/grpc_config.h>
9+
#include <grpc/grpc.h>
10+
11+
using namespace opentelemetry::ext::grpc;
12+
13+
TEST(GrpcConfigTests, OverridesToString)
14+
{
15+
grpc_init();
16+
printf("\n%s\n\n", OverridesToString().c_str());
17+
}
18+
19+
TEST(GrpcConfigTests, ResetThenOverridesToString)
20+
{
21+
grpc_init();
22+
ResetOverrides();
23+
printf("\n%s\n\n", OverridesToString().c_str());
24+
}

‎multitool.lock.json

+18-18
Original file line numberDiff line numberDiff line change
@@ -48,18 +48,18 @@
4848
"binaries": [
4949
{
5050
"kind": "archive",
51-
"url": "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.122.1/otelcol-contrib_0.122.1_windows_amd64.tar.gz",
52-
"file": "otelcol-contrib.exe",
53-
"sha256": "45719e65e5c247e14b252031f3bc59c19afa1bd7503676f1bb821a7d08583eaf",
54-
"os": "windows",
51+
"url": "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.123.1/otelcol-contrib_0.123.1_linux_amd64.tar.gz",
52+
"file": "otelcol-contrib",
53+
"sha256": "5362e106390ff2a35cc472e948e8c894e06277bfc8a1f7cbc49f1ee0aadd0b45",
54+
"os": "linux",
5555
"cpu": "x86_64"
5656
},
5757
{
5858
"kind": "archive",
59-
"url": "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.123.1/otelcol-contrib_0.123.1_linux_amd64.tar.gz",
60-
"sha256": "5362e106390ff2a35cc472e948e8c894e06277bfc8a1f7cbc49f1ee0aadd0b45",
61-
"file": "otelcol-contrib",
62-
"os": "linux",
59+
"url": "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.123.1/otelcol-contrib_0.123.1_windows_amd64.tar.gz",
60+
"file": "otelcol-contrib.exe",
61+
"sha256": "306a8ee06b5ce917cb679ed97eee822372d0a23c213d789035c69dcbb6a39078",
62+
"os": "windows",
6363
"cpu": "x86_64"
6464
}
6565
]
@@ -140,36 +140,36 @@
140140
"binaries": [
141141
{
142142
"kind": "file",
143-
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.42.5/sentry-cli-Linux-aarch64",
144-
"sha256": "2d4f9df6e567ffaf90aab1a6dd892b1ba9fd580edd8b9ffe7d8484e5530263eb",
143+
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.43.0/sentry-cli-Linux-aarch64",
144+
"sha256": "15a596b0ab099c5e42d779e46b76624064a460d6f12bd792af508a5abcd1d293",
145145
"os": "linux",
146146
"cpu": "arm64"
147147
},
148148
{
149149
"kind": "file",
150-
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.42.5/sentry-cli-Linux-x86_64",
151-
"sha256": "ece674996d051b3d236bb3b3e2ae4c315d0aa55373fa3bcde2d6d474b7c6ba96",
150+
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.43.0/sentry-cli-Linux-x86_64",
151+
"sha256": "d9b5d6f98678cbd2d8fe1ade321125cb3a4baaa20463f72f0dc61083184ac85b",
152152
"os": "linux",
153153
"cpu": "x86_64"
154154
},
155155
{
156156
"kind": "file",
157-
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.42.5/sentry-cli-Darwin-arm64",
158-
"sha256": "833a7a87be13f4b28f52928b3890b1112684a8b3d148a3c0cdc0d1e3b65e80f7",
157+
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.43.0/sentry-cli-Darwin-arm64",
158+
"sha256": "e22e2df9d6b8c47f717ab4314aa689493ee8918aeeb04658537c3ec841e62d65",
159159
"os": "macos",
160160
"cpu": "arm64"
161161
},
162162
{
163163
"kind": "file",
164-
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.42.5/sentry-cli-Darwin-x86_64",
165-
"sha256": "5110eb2e6568d836901c3f8f0c02cc596fa36663fab37e2d5f995207e878f8aa",
164+
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.43.0/sentry-cli-Darwin-x86_64",
165+
"sha256": "6744c568dce3651c5f54eaccc209c9372fba93aa9a22027a41a5e36382c80006",
166166
"os": "macos",
167167
"cpu": "x86_64"
168168
},
169169
{
170170
"kind": "file",
171-
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.42.5/sentry-cli-Windows-x86_64.exe",
172-
"sha256": "ea1f5a9ad24ed1ac43c238166857490772fb823d868997e874517648cace13d7",
171+
"url": "https://github.com/getsentry/sentry-cli/releases/download/2.43.0/sentry-cli-Windows-x86_64.exe",
172+
"sha256": "72d42103274135e0e3d0c77dae4632a2f6ebb2e4371b7f9af1aacf15e5d6e19c",
173173
"os": "windows",
174174
"cpu": "x86_64"
175175
}

‎x/MODULE.bazel

+7
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ bazel_dep(name = "gazelle", version = "0.42.0")
3232
bazel_dep(name = "stardoc", version = "0.8.0")
3333
bazel_dep(name = "rules_android", version = "0.6.3")
3434
bazel_dep(name = "grpc", version = "1.71.0")
35+
single_version_override(
36+
module_name = "grpc",
37+
patch_strip = 1,
38+
patches = [
39+
"bazel/grpc_no_absl_sync.patch",
40+
],
41+
)
3542
bazel_dep(name = "nlohmann_json", version = "3.11.3.bcr.1")
3643
bazel_dep(name = "platforms", version = "0.0.11")
3744
bazel_dep(name = "rules_cc", version = "0.1.1")

‎x/bazel/grpc_no_absl_sync.patch

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
diff --git a/include/grpc/support/port_platform.h b/include/grpc/support/port_platform.h
2+
index 04a79ef3c8..6db7822131 100644
3+
--- a/include/grpc/support/port_platform.h
4+
+++ b/include/grpc/support/port_platform.h
5+
@@ -30,7 +30,7 @@
6+
* Defines GPR_ABSEIL_SYNC to use synchronization features from Abseil
7+
*/
8+
#ifndef GPR_ABSEIL_SYNC
9+
-#if defined(__APPLE__)
10+
+#if defined(__APPLE__) || defined(GRPC_NO_ABSL_SYNC)
11+
// This is disabled on Apple platforms because macos/grpc_basictests_c_cpp
12+
// fails with this. https://github.com/grpc/grpc/issues/23661
13+
#else

‎x/hang/BUILD.bazel

+4-6
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
cc_test(
22
name = "hang",
3-
srcs = ["hang.cpp", "otel_sdk.cpp", "otel_api.cpp", "otel_sdk.h", "otel_api.h"],
3+
srcs = ["hang.cpp", "otel_sdk.cpp", "otel_sdk.h"],
44
deps = ["@otel_sdk//:dll"],
5-
env = {
6-
"OTEL_LOG_LEVEL": "5",
7-
"OTEL_ENABLED": "1",
8-
}
9-
)
5+
size = "enormous",
6+
timeout = "eternal",
7+
)

0 commit comments

Comments
 (0)