Skip to content

Commit 74aec3e

Browse files
laramielcopybara-github
laramiel
authored and
copybara-github
committed
internal/http: Update HttpTransport to allow custom certificate use.
Move curl implementation into a subdirectory. For http/2 tests, fork-and-modify example h2 asyncio server. NOKEYCHECK PiperOrigin-RevId: 738189447 Change-Id: I6edbb22248deeed91a34a616dfa0933893aea0eb
1 parent 5ed927b commit 74aec3e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+1733
-1084
lines changed

‎python/tensorstore/BUILD

+1-1
Original file line numberDiff line numberDiff line change
@@ -1207,7 +1207,7 @@ pybind11_cc_library(
12071207
":tensorstore_module_components",
12081208
"//tensorstore/internal:global_initializer",
12091209
"//tensorstore/internal/http",
1210-
"//tensorstore/internal/http:curl_transport",
1210+
"//tensorstore/internal/http:default_transport",
12111211
"//tensorstore/internal/log:verbose_flag",
12121212
"//tensorstore/internal/metrics:collect",
12131213
"//tensorstore/internal/metrics:prometheus",

‎python/tensorstore/experimental.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
#include "python/tensorstore/json_type_caster.h"
3232
#include "python/tensorstore/tensorstore_module_components.h"
3333
#include "tensorstore/internal/global_initializer.h"
34-
#include "tensorstore/internal/http/curl_transport.h"
34+
#include "tensorstore/internal/http/default_transport.h"
3535
#include "tensorstore/internal/http/http_response.h"
3636
#include "tensorstore/internal/http/http_transport.h"
3737
#include "tensorstore/internal/log/verbose_flag.h"

‎tensorstore/internal/cache/BUILD

-1
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,6 @@ tensorstore_cc_test(
149149
":kvs_backed_cache_testutil",
150150
"//tensorstore:transaction",
151151
"//tensorstore/internal:global_initializer",
152-
"//tensorstore/kvstore",
153152
"//tensorstore/kvstore:generation",
154153
"//tensorstore/kvstore:key_range",
155154
"//tensorstore/kvstore:mock_kvstore",

‎tensorstore/internal/curl/BUILD

+166
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
load("//bazel:tensorstore.bzl", "tensorstore_cc_library", "tensorstore_cc_test")
2+
3+
package(default_visibility = ["//tensorstore:internal_packages"])
4+
5+
licenses(["notice"])
6+
7+
# Ensure that ws2_32.lib is linked for mingw.
8+
_WS2_32_LINKOPTS = select({
9+
"@tensorstore//:compiler_mingw_gcc": [
10+
"-DEFAULTLIB:ws2_32.lib",
11+
"-lws2_32",
12+
],
13+
"@tensorstore//:compiler_msvc": ["-DEFAULTLIB:ws2_32.lib"],
14+
"//conditions:default": [],
15+
})
16+
17+
tensorstore_cc_library(
18+
name = "curl_factory",
19+
srcs = ["curl_factory_hook.cc"],
20+
hdrs = ["curl_factory.h"],
21+
deps = [
22+
":curl_wrappers",
23+
"//tensorstore/internal/os:file_util",
24+
"@com_google_absl//absl/log:absl_check",
25+
"@com_google_boringssl//:crypto",
26+
"@se_curl//:curl",
27+
],
28+
)
29+
30+
tensorstore_cc_library(
31+
name = "curl_handle",
32+
srcs = ["curl_handle.cc"],
33+
hdrs = ["curl_handle.h"],
34+
deps = [
35+
":curl_factory",
36+
":curl_wrappers",
37+
"//tensorstore/internal:source_location",
38+
"@com_google_absl//absl/base:core_headers",
39+
"@com_google_absl//absl/log:absl_check",
40+
"@se_curl//:curl",
41+
],
42+
)
43+
44+
tensorstore_cc_library(
45+
name = "curl_transport",
46+
srcs = ["curl_transport.cc"],
47+
hdrs = ["curl_transport.h"],
48+
deps = [
49+
":curl_factory",
50+
":curl_handle",
51+
":curl_wrappers",
52+
":default_factory",
53+
"//tensorstore/internal:cord_util",
54+
"//tensorstore/internal:env",
55+
"//tensorstore/internal/container:circular_queue",
56+
"//tensorstore/internal/http",
57+
"//tensorstore/internal/metrics",
58+
"//tensorstore/internal/metrics:metadata",
59+
"//tensorstore/internal/thread",
60+
"@com_google_absl//absl/flags:flag",
61+
"@com_google_absl//absl/log:absl_log",
62+
"@com_google_absl//absl/status",
63+
"@com_google_absl//absl/strings",
64+
"@com_google_absl//absl/strings:cord",
65+
"@com_google_absl//absl/synchronization",
66+
"@com_google_absl//absl/time",
67+
"@se_curl//:curl",
68+
],
69+
alwayslink = True,
70+
)
71+
72+
tensorstore_cc_test(
73+
name = "curl_transport_test",
74+
srcs = ["curl_transport_test.cc"],
75+
linkopts = _WS2_32_LINKOPTS,
76+
deps = [
77+
":curl_transport",
78+
"//tensorstore/internal/http",
79+
"//tensorstore/internal/http:transport_test_utils",
80+
"//tensorstore/internal/thread",
81+
"@com_google_absl//absl/log:absl_check",
82+
"@com_google_absl//absl/log:absl_log",
83+
"@com_google_absl//absl/strings",
84+
"@com_google_absl//absl/strings:cord",
85+
"@com_google_googletest//:gtest_main",
86+
],
87+
)
88+
89+
tensorstore_cc_library(
90+
name = "curl_wrappers",
91+
srcs = ["curl_wrappers.cc"],
92+
hdrs = ["curl_wrappers.h"],
93+
deps = [
94+
"//tensorstore/internal:source_location",
95+
"//tensorstore/util:status",
96+
"//tensorstore/util:str_cat",
97+
"@com_google_absl//absl/base",
98+
"@com_google_absl//absl/log:absl_check",
99+
"@com_google_absl//absl/log:absl_log",
100+
"@com_google_absl//absl/status",
101+
"@com_google_absl//absl/strings:cord",
102+
"@se_curl//:curl",
103+
],
104+
)
105+
106+
tensorstore_cc_test(
107+
name = "curl_wrappers_test",
108+
size = "small",
109+
srcs = ["curl_wrappers_test.cc"],
110+
deps = [
111+
":curl_wrappers",
112+
"@com_google_absl//absl/status",
113+
"@com_google_googletest//:gtest_main",
114+
"@se_curl//:curl",
115+
],
116+
)
117+
118+
tensorstore_cc_library(
119+
name = "default_factory",
120+
srcs = ["default_factory.cc"],
121+
hdrs = ["default_factory.h"],
122+
deps = [
123+
":curl_factory",
124+
":curl_wrappers",
125+
"//tensorstore/internal:env",
126+
"//tensorstore/internal/log:verbose_flag",
127+
"@com_google_absl//absl/base:core_headers",
128+
"@com_google_absl//absl/flags:flag",
129+
"@com_google_absl//absl/log:absl_check",
130+
"@com_google_absl//absl/log:absl_log",
131+
"@se_curl//:curl",
132+
],
133+
)
134+
135+
tensorstore_cc_test(
136+
name = "http2_test",
137+
srcs = ["http2_test.cc"],
138+
args = [
139+
"--test_httpserver_binary=$(location //tensorstore/internal/http/py:h2_server)",
140+
],
141+
data = ["//tensorstore/internal/http/py:h2_server"],
142+
flaky = 1,
143+
tags = [
144+
"cpu:2",
145+
"requires-net:loopback",
146+
"skip-cmake",
147+
# NOTE: This test can fail on darwin/windows with the error:
148+
# CURL error Unsupported protocol: Received HTTP/0.9 when not allowed
149+
# Which might be a problem with the custom certificate.
150+
"skip-darwin",
151+
"skip-windows",
152+
],
153+
deps = [
154+
":curl_transport",
155+
":default_factory",
156+
"//tensorstore/internal/http",
157+
"//tensorstore/internal/http:test_httpserver",
158+
"@com_google_absl//absl/base",
159+
"@com_google_absl//absl/base:no_destructor",
160+
"@com_google_absl//absl/log:absl_log",
161+
"@com_google_absl//absl/strings",
162+
"@com_google_absl//absl/strings:str_format",
163+
"@com_google_absl//absl/time",
164+
"@com_google_googletest//:gtest_main",
165+
],
166+
)

‎tensorstore/internal/http/curl_factory.h ‎tensorstore/internal/curl/curl_factory.h

+4-9
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,10 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
#ifndef TENSORSTORE_INTERNAL_HTTP_CURL_FACTORY_H_
16-
#define TENSORSTORE_INTERNAL_HTTP_CURL_FACTORY_H_
15+
#ifndef TENSORSTORE_INTERNAL_CURL_CURL_FACTORY_H_
16+
#define TENSORSTORE_INTERNAL_CURL_CURL_FACTORY_H_
1717

18-
#include <memory>
19-
20-
#include "tensorstore/internal/http/curl_wrappers.h"
18+
#include "tensorstore/internal/curl/curl_wrappers.h"
2119

2220
namespace tensorstore {
2321
namespace internal_http {
@@ -38,13 +36,10 @@ class CurlHandleFactory {
3836
virtual void CleanupMultiHandle(CurlMulti&&) = 0;
3937
};
4038

41-
/// Returns the default CurlHandleFactory.
42-
std::shared_ptr<CurlHandleFactory> GetDefaultCurlHandleFactory();
43-
4439
/// Extensibility hooks for libcurl handles.
4540
void CurlPtrHook(CurlPtr& handle);
4641

4742
} // namespace internal_http
4843
} // namespace tensorstore
4944

50-
#endif // TENSORSTORE_INTERNAL_HTTP_CURL_FACTORY_H_
45+
#endif // TENSORSTORE_INTERNAL_CURL_CURL_FACTORY_H_

‎tensorstore/internal/http/curl_factory_hook.cc ‎tensorstore/internal/curl/curl_factory_hook.cc

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020
#include "absl/flags/flag.h"
2121
#include "absl/log/absl_check.h"
2222
#include <curl/curl.h> // IWYU pragma: keep
23+
#include "tensorstore/internal/curl/curl_factory.h"
24+
#include "tensorstore/internal/curl/curl_wrappers.h"
2325
#include "tensorstore/internal/env.h"
24-
#include "tensorstore/internal/http/curl_factory.h"
25-
#include "tensorstore/internal/http/curl_wrappers.h"
2626
#include "tensorstore/internal/os/file_util.h"
2727

2828
#ifndef _WIN32

‎tensorstore/internal/http/curl_handle.cc ‎tensorstore/internal/curl/curl_handle.cc

+6-2
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,16 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
#include "tensorstore/internal/http/curl_handle.h"
15+
#include "tensorstore/internal/curl/curl_handle.h"
1616

1717
#include <stdint.h>
1818

19+
#include <utility>
20+
21+
#include "absl/base/macros.h"
1922
#include <curl/curl.h>
20-
#include "tensorstore/internal/http/curl_factory.h"
23+
#include "tensorstore/internal/curl/curl_factory.h"
24+
#include "tensorstore/internal/curl/curl_wrappers.h"
2125

2226
namespace tensorstore {
2327
namespace internal_http {

‎tensorstore/internal/http/curl_handle.h ‎tensorstore/internal/curl/curl_handle.h

+5-5
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,17 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
#ifndef TENSORSTORE_INTERNAL_HTTP_CURL_HANDLE_H_
16-
#define TENSORSTORE_INTERNAL_HTTP_CURL_HANDLE_H_
15+
#ifndef TENSORSTORE_INTERNAL_CURL_CURL_HANDLE_H_
16+
#define TENSORSTORE_INTERNAL_CURL_CURL_HANDLE_H_
1717

1818
#include <stddef.h>
1919
#include <stdint.h>
2020

2121
#include <cstddef> // std::nullptr_t
2222

2323
#include "absl/log/absl_check.h"
24-
#include "tensorstore/internal/http/curl_factory.h"
25-
#include "tensorstore/internal/http/curl_wrappers.h"
24+
#include "tensorstore/internal/curl/curl_factory.h"
25+
#include "tensorstore/internal/curl/curl_wrappers.h"
2626
#include "tensorstore/internal/source_location.h"
2727

2828
namespace tensorstore {
@@ -90,4 +90,4 @@ class CurlHandle {
9090
} // namespace internal_http
9191
} // namespace tensorstore
9292

93-
#endif // TENSORSTORE_INTERNAL_HTTP_CURL_HANDLE_H_
93+
#endif // TENSORSTORE_INTERNAL_CURL_CURL_HANDLE_H_

‎tensorstore/internal/http/curl_transport.cc ‎tensorstore/internal/curl/curl_transport.cc

+10-42
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
#include "tensorstore/internal/http/curl_transport.h"
15+
#include "tensorstore/internal/curl/curl_transport.h"
1616

1717
#include <stddef.h>
1818
#include <stdint.h>
@@ -31,8 +31,6 @@
3131
#include <utility>
3232
#include <vector>
3333

34-
#include "absl/base/attributes.h"
35-
#include "absl/base/const_init.h"
3634
#include "absl/flags/flag.h"
3735
#include "absl/log/absl_log.h"
3836
#include "absl/status/status.h"
@@ -44,10 +42,11 @@
4442
#include <curl/curl.h>
4543
#include "tensorstore/internal/container/circular_queue.h"
4644
#include "tensorstore/internal/cord_util.h"
45+
#include "tensorstore/internal/curl/curl_factory.h"
46+
#include "tensorstore/internal/curl/curl_handle.h"
47+
#include "tensorstore/internal/curl/curl_wrappers.h"
48+
#include "tensorstore/internal/curl/default_factory.h"
4749
#include "tensorstore/internal/env.h"
48-
#include "tensorstore/internal/http/curl_factory.h"
49-
#include "tensorstore/internal/http/curl_handle.h"
50-
#include "tensorstore/internal/http/curl_wrappers.h"
5150
#include "tensorstore/internal/http/http_request.h"
5251
#include "tensorstore/internal/http/http_transport.h"
5352
#include "tensorstore/internal/metrics/counter.h"
@@ -464,7 +463,7 @@ void MultiTransportImpl::FinishRequest(std::unique_ptr<CurlRequestState> state,
464463
}
465464

466465
if (code != CURLE_OK) {
467-
/// Transfer failed; set the status
466+
// Transfer failed; set the status
468467
state->response_handler_->OnFailure(
469468
CurlCodeToStatus(code, state->error_buffer_));
470469
return;
@@ -599,41 +598,10 @@ void CurlTransport::IssueRequestWithHandler(
599598
impl_->EnqueueRequest(request, std::move(options), response_handler);
600599
}
601600

602-
namespace {
603-
struct GlobalTransport {
604-
std::shared_ptr<HttpTransport> transport_;
605-
606-
std::shared_ptr<HttpTransport> Get() {
607-
if (!transport_) {
608-
transport_ =
609-
std::make_shared<CurlTransport>(GetDefaultCurlHandleFactory());
610-
}
611-
return transport_;
612-
}
613-
614-
void Set(std::shared_ptr<HttpTransport> transport) {
615-
transport_ = std::move(transport);
616-
}
617-
};
618-
619-
ABSL_CONST_INIT absl::Mutex global_mu(absl::kConstInit);
620-
621-
static GlobalTransport& GetGlobalTransport() {
622-
static auto* g = new GlobalTransport();
623-
return *g;
624-
}
625-
626-
} // namespace
627-
628-
std::shared_ptr<HttpTransport> GetDefaultHttpTransport() {
629-
absl::MutexLock l(&global_mu);
630-
return GetGlobalTransport().Get();
631-
}
632-
633-
/// Sets the default CurlMultiTransport. Exposed for test mocking.
634-
void SetDefaultHttpTransport(std::shared_ptr<HttpTransport> t) {
635-
absl::MutexLock l(&global_mu);
636-
return GetGlobalTransport().Set(std::move(t));
601+
std::shared_ptr<HttpTransport> GetDefaultCurlTransport() {
602+
static std::shared_ptr<HttpTransport> transport =
603+
std::make_shared<CurlTransport>(GetDefaultCurlHandleFactory());
604+
return transport;
637605
}
638606

639607
} // namespace internal_http

0 commit comments

Comments
 (0)