Skip to content

Commit 65e5f95

Browse files
rawalexecookpate
rawalexe
and
cookpate
authored
Add support for http retry on failure (#825)
Co-authored-by: Patrick Cook <[email protected]>
1 parent 1ff256a commit 65e5f95

File tree

2 files changed

+175
-49
lines changed

2 files changed

+175
-49
lines changed

Diff for: ‎modules/ggl-http/CMakeLists.txt

+9-2
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,12 @@
22
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
33
# SPDX-License-Identifier: Apache-2.0
44

5-
ggl_init_module(ggl-http LIBS ggl-lib ggl-file aws_sigv4 PkgConfig::openssl
6-
PkgConfig::libcurl)
5+
ggl_init_module(
6+
ggl-http
7+
LIBS ggl-backoff
8+
ggl-lib
9+
ggl-file
10+
aws_sigv4
11+
PkgConfig::openssl
12+
PkgConfig::liburiparser
13+
PkgConfig::libcurl)

Diff for: ‎modules/ggl-http/src/gghttp_util.c

+166-47
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,15 @@
99
#include <sys/types.h>
1010
#include <assert.h>
1111
#include <curl/curl.h>
12+
#include <errno.h>
13+
#include <ggl/backoff.h>
1214
#include <ggl/cleanup.h>
1315
#include <ggl/file.h>
1416
#include <ggl/log.h>
1517
#include <ggl/vector.h>
1618
#include <pthread.h>
19+
#include <unistd.h>
20+
#include <stdbool.h>
1721
#include <stdlib.h>
1822

1923
#define MAX_HEADER_LENGTH 1024
@@ -47,6 +51,162 @@ static GglError translate_curl_code(CURLcode code) {
4751
}
4852
}
4953

54+
static bool can_retry(CURLcode code, CurlData *data) {
55+
switch (code) {
56+
// If OK, then inspect HTTP status code.
57+
case CURLE_OK:
58+
break;
59+
60+
case CURLE_OPERATION_TIMEDOUT:
61+
case CURLE_COULDNT_CONNECT:
62+
case CURLE_SSL_CONNECT_ERROR:
63+
case CURLE_GOT_NOTHING:
64+
case CURLE_SEND_ERROR:
65+
case CURLE_RECV_ERROR:
66+
case CURLE_PARTIAL_FILE:
67+
case CURLE_AGAIN:
68+
return true;
69+
70+
default:
71+
return false;
72+
}
73+
74+
long http_status_code = 0;
75+
curl_easy_getinfo(data->curl, CURLINFO_HTTP_CODE, &http_status_code);
76+
77+
switch (http_status_code) {
78+
case 400: // Generic client error
79+
case 408: // Request timeout
80+
// TODO: 429 can contain a retry-after header.
81+
// This should be used as the backoff.
82+
// Also add a upper limit to retry-after
83+
case 429: // Too many requests
84+
case 500: // Generic server error
85+
case 502: // Bad gateway
86+
case 503: // Service unavailable
87+
case 504: // Gateway Timeout
88+
case 509: // Server bandwidth exceeded
89+
return true;
90+
default:
91+
return false;
92+
}
93+
}
94+
95+
typedef struct CurlRequestRetryCtx {
96+
CurlData *curl_data;
97+
98+
// reset response_data for next attempt
99+
GglError (*retry_fn)(void *);
100+
void *response_data;
101+
102+
// Needed to propagate errors when retrying is impossible.
103+
GglError err;
104+
} CurlRequestRetryCtx;
105+
106+
static GglError clear_buffer(void *response_data) {
107+
GglByteVec *vector = (GglByteVec *) response_data;
108+
vector->buf.len = 0;
109+
return GGL_ERR_OK;
110+
}
111+
112+
static GglError truncate_file(void *response_data) {
113+
int fd = *(int *) response_data;
114+
115+
int ret;
116+
do {
117+
ret = ftruncate(fd, 0);
118+
} while ((ret == -1) && (errno == EINTR));
119+
120+
if (ret == -1) {
121+
GGL_LOGE("Failed to truncate fd for write (errno=%d).", errno);
122+
return GGL_ERR_FAILURE;
123+
}
124+
return GGL_ERR_OK;
125+
}
126+
127+
static GglError curl_request_retry_wrapper(void *ctx) {
128+
CurlRequestRetryCtx *retry_ctx = (CurlRequestRetryCtx *) ctx;
129+
CurlData *curl_data = retry_ctx->curl_data;
130+
131+
CURLcode curl_error = curl_easy_perform(curl_data->curl);
132+
if (can_retry(curl_error, curl_data)) {
133+
GglError err = retry_ctx->retry_fn(retry_ctx->response_data);
134+
if (err != GGL_ERR_OK) {
135+
retry_ctx->err = err;
136+
return GGL_ERR_OK;
137+
}
138+
return GGL_ERR_FAILURE;
139+
}
140+
if (curl_error != CURLE_OK) {
141+
GGL_LOGE(
142+
"Curl request failed due to error: %s",
143+
curl_easy_strerror(curl_error)
144+
);
145+
retry_ctx->err = translate_curl_code(curl_error);
146+
return GGL_ERR_OK;
147+
}
148+
int long http_status_code = 0;
149+
curl_error = curl_easy_getinfo(
150+
curl_data->curl, CURLINFO_HTTP_CODE, &http_status_code
151+
);
152+
if (curl_error != CURLE_OK) {
153+
retry_ctx->err = GGL_ERR_FAILURE;
154+
return GGL_ERR_OK;
155+
}
156+
157+
if ((http_status_code >= 200) && (http_status_code < 300)) {
158+
retry_ctx->err = GGL_ERR_OK;
159+
return GGL_ERR_OK;
160+
}
161+
162+
if ((http_status_code >= 500) && (http_status_code < 600)) {
163+
retry_ctx->err = GGL_ERR_REMOTE;
164+
} else {
165+
retry_ctx->err = GGL_ERR_FAILURE;
166+
}
167+
GGL_LOGE(
168+
"Curl request failed due to HTTP status code %ld.", http_status_code
169+
);
170+
return GGL_ERR_OK;
171+
}
172+
173+
static GglError do_curl_request(
174+
CurlData *curl_data, GglByteVec *response_buffer
175+
) {
176+
CurlRequestRetryCtx ctx = { .curl_data = curl_data,
177+
.response_data = (void *) response_buffer,
178+
.retry_fn = clear_buffer,
179+
.err = GGL_ERR_OK };
180+
GglError ret = ggl_backoff(
181+
1000, 64000, 7, curl_request_retry_wrapper, (void *) &ctx
182+
);
183+
if (ret != GGL_ERR_OK) {
184+
return ret;
185+
}
186+
if (ctx.err != GGL_ERR_OK) {
187+
return ctx.err;
188+
}
189+
return GGL_ERR_OK;
190+
}
191+
192+
static GglError do_curl_request_fd(CurlData *curl_data, int fd) {
193+
CurlRequestRetryCtx ctx = { .curl_data = curl_data,
194+
.response_data = (void *) &fd,
195+
.retry_fn = truncate_file,
196+
.err = GGL_ERR_OK };
197+
GglError ret = ggl_backoff(
198+
1000, 64000, 7, curl_request_retry_wrapper, (void *) &ctx
199+
);
200+
if (ret != GGL_ERR_OK) {
201+
GGL_LOGE("Curl request failed; retries exhausted.");
202+
return ret;
203+
}
204+
if (ctx.err != GGL_ERR_OK) {
205+
return ctx.err;
206+
}
207+
return GGL_ERR_OK;
208+
}
209+
50210
/// @brief Callback function to write the HTTP response data to a buffer.
51211
///
52212
/// This function is used as a callback by CURL to handle the response data
@@ -137,6 +297,7 @@ GglError gghttplib_init_curl(CurlData *curl_data, const char *url) {
137297
}
138298

139299
CURLcode err = curl_easy_setopt(curl_data->curl, CURLOPT_URL, url);
300+
140301
return translate_curl_code(err);
141302
}
142303

@@ -278,41 +439,11 @@ GglError gghttplib_process_request(
278439
}
279440
}
280441

281-
curl_error = curl_easy_perform(curl_data->curl);
282-
if (curl_error != CURLE_OK) {
283-
GGL_LOGE(
284-
"curl_easy_perform() failed: %s", curl_easy_strerror(curl_error)
285-
);
286-
if (curl_error == CURLE_WRITE_ERROR) {
287-
return GGL_ERR_NOMEM;
288-
}
289-
return translate_curl_code(curl_error);
290-
}
291-
292-
long http_status_code = 0;
293-
curl_error = curl_easy_getinfo(
294-
curl_data->curl, CURLINFO_HTTP_CODE, &http_status_code
295-
);
296-
297-
if (curl_error != CURLE_OK) {
298-
GGL_LOGE(
299-
"curl_easy_getinfo() failed: %s", curl_easy_strerror(curl_error)
300-
);
301-
return translate_curl_code(curl_error);
302-
}
303-
304-
GGL_LOGI("HTTP code: %ld", http_status_code);
305-
306-
if (response_buffer != NULL) {
442+
GglError ret = do_curl_request(curl_data, &response_vector);
443+
if ((response_buffer != NULL) && (ret == GGL_ERR_OK)) {
307444
response_buffer->len = response_vector.buf.len;
308445
}
309-
310-
// TODO: propagate HTTP code up for deployment failure root causing
311-
if (http_status_code < 200 || http_status_code > 299) {
312-
return GGL_ERR_FAILURE;
313-
}
314-
315-
return translate_curl_code(curl_error);
446+
return ret;
316447
}
317448

318449
GglError gghttplib_process_request_with_fd(CurlData *curl_data, int fd) {
@@ -328,6 +459,7 @@ GglError gghttplib_process_request_with_fd(CurlData *curl_data, int fd) {
328459
if (curl_error != CURLE_OK) {
329460
return translate_curl_code(curl_error);
330461
}
462+
331463
// coverity[bad_sizeof]
332464
curl_error
333465
= curl_easy_setopt(curl_data->curl, CURLOPT_WRITEDATA, (void *) &fd);
@@ -339,18 +471,5 @@ GglError gghttplib_process_request_with_fd(CurlData *curl_data, int fd) {
339471
return translate_curl_code(curl_error);
340472
}
341473

342-
curl_error = curl_easy_perform(curl_data->curl);
343-
if (curl_error != CURLE_OK) {
344-
GGL_LOGE(
345-
"curl_easy_perform() failed: %s", curl_easy_strerror(curl_error)
346-
);
347-
return translate_curl_code(curl_error);
348-
}
349-
350-
long http_status_code = 0;
351-
curl_easy_getinfo(curl_data->curl, CURLINFO_HTTP_CODE, &http_status_code);
352-
GGL_LOGI("HTTP code: %ld", http_status_code);
353-
354-
// TODO: propagate HTTP code up for deployment failure root causing
355-
return translate_curl_code(curl_error);
474+
return do_curl_request_fd(curl_data, fd);
356475
}

0 commit comments

Comments
 (0)