Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for http retry on failure #825

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+174 −49
Open

Add support for http retry on failure #825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9bf5596
Add support for http retry on failure
rawalexe Mar 28, 2025
f0f06bb
Use backoff algorithm in ggl-http
cookpate Mar 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions modules/ggl-http/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,12 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

ggl_init_module(ggl-http LIBS ggl-lib ggl-file aws_sigv4 PkgConfig::openssl
PkgConfig::liburiparser PkgConfig::libcurl)
ggl_init_module(
ggl-http
LIBS ggl-backoff
ggl-lib
ggl-file
aws_sigv4
PkgConfig::openssl
PkgConfig::liburiparser
PkgConfig::libcurl)
212 changes: 165 additions & 47 deletions modules/ggl-http/src/gghttp_util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@
#include <sys/types.h>
#include <assert.h>
#include <curl/curl.h>
#include <errno.h>
#include <ggl/backoff.h>
#include <ggl/cleanup.h>
#include <ggl/file.h>
#include <ggl/log.h>
#include <ggl/vector.h>
#include <pthread.h>
#include <unistd.h>
#include <stdbool.h>
#include <stdlib.h>

#define MAX_HEADER_LENGTH 1024
Expand Down Expand Up @@ -47,6 +51,161 @@ static GglError translate_curl_code(CURLcode code) {
}
}

static bool can_retry(CURLcode code, CurlData *data) {
switch (code) {
// If OK, then inspect HTTP status code.
case CURLE_OK:
break;

case CURLE_OPERATION_TIMEDOUT:
case CURLE_COULDNT_CONNECT:
case CURLE_SSL_CONNECT_ERROR:
case CURLE_GOT_NOTHING:
case CURLE_SEND_ERROR:
case CURLE_RECV_ERROR:
case CURLE_PARTIAL_FILE:
case CURLE_AGAIN:
return true;

default:
return false;
}

long http_status_code = 0;
curl_easy_getinfo(data->curl, CURLINFO_HTTP_CODE, &http_status_code);

switch (http_status_code) {
case 400: // Generic client error
case 408: // Request timeout
// TODO: 429 can contain a retry-after header.
// This should be used as the backoff.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if retry-after is like 2 days, we dont want to block that long. Probably would need to cap the accepted value.

case 429: // Too many requests
case 500: // Generic server error
case 502: // Bad gateway
case 503: // Service unavailable
case 504: // Gateway Timeout
case 509: // Server bandwidth exceeded
return true;
default:
return false;
}
}

typedef struct CurlRequestRetryCtx {
CurlData *curl_data;

// reset response_data for next attempt
GglError (*retry_fn)(void *);
void *response_data;

// Needed to propagate errors when retrying is impossible.
GglError err;
} CurlRequestRetryCtx;

static GglError clear_buffer(void *response_data) {
GglByteVec *vector = (GglByteVec *) response_data;
vector->buf.len = 0;
return GGL_ERR_OK;
}

static GglError truncate_file(void *response_data) {
int fd = *(int *) response_data;

int ret;
do {
ret = ftruncate(fd, 0);
} while ((ret == -1) && (errno == EINTR));

if (ret == -1) {
GGL_LOGE("Failed to truncate fd for write (errno=%d).", errno);
return GGL_ERR_FAILURE;
}
return GGL_ERR_OK;
}

static GglError curl_request_retry_wrapper(void *ctx) {
CurlRequestRetryCtx *retry_ctx = (CurlRequestRetryCtx *) ctx;
CurlData *curl_data = retry_ctx->curl_data;

CURLcode curl_error = curl_easy_perform(curl_data->curl);
if (can_retry(curl_error, curl_data)) {
GglError err = retry_ctx->retry_fn(retry_ctx->response_data);
if (err != GGL_ERR_OK) {
retry_ctx->err = err;
return GGL_ERR_OK;
}
return GGL_ERR_FAILURE;
}
if (curl_error != CURLE_OK) {
GGL_LOGE(
"Curl request failed due to error: %s",
curl_easy_strerror(curl_error)
);
retry_ctx->err = translate_curl_code(curl_error);
return GGL_ERR_OK;
}
int long http_status_code = 0;
curl_error = curl_easy_getinfo(
curl_data->curl, CURLINFO_HTTP_CODE, &http_status_code
);
if (curl_error != CURLE_OK) {
retry_ctx->err = GGL_ERR_FAILURE;
return GGL_ERR_OK;
}

if ((http_status_code >= 200) && (http_status_code < 300)) {
retry_ctx->err = GGL_ERR_OK;
return GGL_ERR_OK;
}

if ((http_status_code >= 500) && (http_status_code < 600)) {
retry_ctx->err = GGL_ERR_REMOTE;
} else {
retry_ctx->err = GGL_ERR_FAILURE;
}
GGL_LOGE(
"Curl request failed due to HTTP status code %ld.", http_status_code
);
return GGL_ERR_OK;
}

static GglError do_curl_request(
CurlData *curl_data, GglByteVec *response_buffer
) {
CurlRequestRetryCtx ctx = { .curl_data = curl_data,
.response_data = (void *) response_buffer,
.retry_fn = clear_buffer,
.err = GGL_ERR_OK };
GglError ret = ggl_backoff(
1000, 64000, 7, curl_request_retry_wrapper, (void *) &ctx
);
if (ret != GGL_ERR_OK) {
return ret;
}
if (ctx.err != GGL_ERR_OK) {
return ctx.err;
}
return GGL_ERR_OK;
}

static GglError do_curl_request_fd(CurlData *curl_data, int fd) {
CurlRequestRetryCtx ctx = { .curl_data = curl_data,
.response_data = (void *) &fd,
.retry_fn = truncate_file,
.err = GGL_ERR_OK };
GglError ret = ggl_backoff(
1000, 64000, 7, curl_request_retry_wrapper, (void *) &ctx
);
if (ret != GGL_ERR_OK) {
GGL_LOGE("Curl request failed; retries exhausted.");
return ret;
}
if (ctx.err != GGL_ERR_OK) {
return ctx.err;
}
return GGL_ERR_OK;
}

/// @brief Callback function to write the HTTP response data to a buffer.
///
/// This function is used as a callback by CURL to handle the response data
Expand Down Expand Up @@ -137,6 +296,7 @@ GglError gghttplib_init_curl(CurlData *curl_data, const char *url) {
}

CURLcode err = curl_easy_setopt(curl_data->curl, CURLOPT_URL, url);

return translate_curl_code(err);
}

Expand Down Expand Up @@ -278,41 +438,11 @@ GglError gghttplib_process_request(
}
}

curl_error = curl_easy_perform(curl_data->curl);
if (curl_error != CURLE_OK) {
GGL_LOGE(
"curl_easy_perform() failed: %s", curl_easy_strerror(curl_error)
);
if (curl_error == CURLE_WRITE_ERROR) {
return GGL_ERR_NOMEM;
}
return translate_curl_code(curl_error);
}

long http_status_code = 0;
curl_error = curl_easy_getinfo(
curl_data->curl, CURLINFO_HTTP_CODE, &http_status_code
);

if (curl_error != CURLE_OK) {
GGL_LOGE(
"curl_easy_getinfo() failed: %s", curl_easy_strerror(curl_error)
);
return translate_curl_code(curl_error);
}

GGL_LOGI("HTTP code: %ld", http_status_code);

if (response_buffer != NULL) {
GglError ret = do_curl_request(curl_data, &response_vector);
if ((response_buffer != NULL) && (ret == GGL_ERR_OK)) {
response_buffer->len = response_vector.buf.len;
}

// TODO: propagate HTTP code up for deployment failure root causing
if (http_status_code < 200 || http_status_code > 299) {
return GGL_ERR_FAILURE;
}

return translate_curl_code(curl_error);
return ret;
}

GglError gghttplib_process_request_with_fd(CurlData *curl_data, int fd) {
Expand All @@ -328,6 +458,7 @@ GglError gghttplib_process_request_with_fd(CurlData *curl_data, int fd) {
if (curl_error != CURLE_OK) {
return translate_curl_code(curl_error);
}

// coverity[bad_sizeof]
curl_error
= curl_easy_setopt(curl_data->curl, CURLOPT_WRITEDATA, (void *) &fd);
Expand All @@ -339,18 +470,5 @@ GglError gghttplib_process_request_with_fd(CurlData *curl_data, int fd) {
return translate_curl_code(curl_error);
}

curl_error = curl_easy_perform(curl_data->curl);
if (curl_error != CURLE_OK) {
GGL_LOGE(
"curl_easy_perform() failed: %s", curl_easy_strerror(curl_error)
);
return translate_curl_code(curl_error);
}

long http_status_code = 0;
curl_easy_getinfo(curl_data->curl, CURLINFO_HTTP_CODE, &http_status_code);
GGL_LOGI("HTTP code: %ld", http_status_code);

// TODO: propagate HTTP code up for deployment failure root causing
return translate_curl_code(curl_error);
return do_curl_request_fd(curl_data, fd);
}
Loading