Retry S3 download on 403 error

Author: rawalexe

Diff for: modules/ggdeploymentd/src/deployment_handler.c

@@ -14,6 +14,7 @@
 #include <assert.h>
 #include <fcntl.h>
 #include <ggl/alloc.h>
+#include <ggl/backoff.h>
 #include <ggl/base64.h>
 #include <ggl/buffer.h>
 #include <ggl/bump_alloc.h>
@@ -409,6 +410,84 @@ static GglError get_tes_credentials(TesCredentials *tes_creds) {
     return GGL_ERR_OK;
 }
 
+typedef struct CurlRequestRetryCtx {
+    const char *url_for_sigv4_download;
+    GglBuffer host;
+    GglBuffer file_path;
+    SigV4Details sigv4_details;
+
+    // reset response_data for next attempt
+    GglError (*retry_cleanup_fn)(void *);
+    void *response_data;
+
+    // Needed to propagate errors when retrying is impossible.
+    GglError err;
+} DownloadRequestRetryCtx;
+
+static GglError retry_download_wrapper(void *ctx) {
+    DownloadRequestRetryCtx *retry_ctx = (DownloadRequestRetryCtx *) ctx;
+    long http_response_code;
+
+    GglError ret = sigv4_download(
+        retry_ctx->url_for_sigv4_download,
+        retry_ctx->host,
+        retry_ctx->file_path,
+        *(int *) retry_ctx->response_data,
+        retry_ctx->sigv4_details,
+        &http_response_code
+    );
+    if (http_response_code == (long) 403) {
+        GglError err = retry_ctx->retry_cleanup_fn(retry_ctx->response_data);
+        GGL_LOGE(
+            "Artifact download attempt failed with 403. Retrying with backoff."
+        );
+        if (err != GGL_ERR_OK) {
+            retry_ctx->err = err;
+            return GGL_ERR_OK;
+        }
+        return GGL_ERR_FAILURE;
+    }
+    if (ret != GGL_ERR_OK) {
+        GGL_LOGE(
+            "Artifact download attempt failed due to error: %d", ret
+
+        );
+        retry_ctx->err = ret;
+        return GGL_ERR_OK;
+    }
+
+    retry_ctx->err = ret;
+    return GGL_ERR_OK;
+}
+
+static GglError retryable_download_request(
+    const char *url_for_sigv4_download,
+    GglBuffer host,
+    GglBuffer file_path,
+    int artifact_fd,
+    SigV4Details sigv4_details
+) {
+    DownloadRequestRetryCtx ctx
+        = { .url_for_sigv4_download = url_for_sigv4_download,
+            .host = host,
+            .file_path = file_path,
+            .sigv4_details = sigv4_details,
+            .response_data = (void *) &artifact_fd,
+            .retry_cleanup_fn = truncate_file,
+            .err = GGL_ERR_OK };
+
+    GglError ret
+        = ggl_backoff(3000, 64000, 3, retry_download_wrapper, (void *) &ctx);
+    if (ret != GGL_ERR_OK) {
+        GGL_LOGE("Artifact download attempt failed; retries exhausted.");
+        return ret;
+    }
+    if (ctx.err != GGL_ERR_OK) {
+        return ctx.err;
+    }
+    return GGL_ERR_OK;
+}
+
 static GglError download_s3_artifact(
     GglBuffer scratch_buffer,
     GglUriInfo uri_info,
@@ -434,7 +513,7 @@ static GglError download_s3_artifact(
         return error;
     }
 
-    return sigv4_download(
+    return retryable_download_request(
         (const char *) url_vec.buf.data,
         (GglBuffer) { .data = &scratch_buffer.data[start_loc],
                       .len = end_loc - start_loc },

Diff for: modules/ggl-file/include/ggl/file.h

@@ -87,4 +87,6 @@ static inline void cleanup_closedir(DIR **dirp) {
     }
 }
 
+GglError truncate_file(void *response_data);
+
 #endif

Diff for: modules/ggl-file/src/file.c

@@ -578,3 +578,18 @@ GglError ggl_file_read_path(GglBuffer path, GglBuffer *content) {
 
     return GGL_ERR_OK;
 }
+
+GglError truncate_file(void *response_data) {
+    int fd = *(int *) response_data;
+
+    int ret;
+    do {
+        ret = ftruncate(fd, 0);
+    } while ((ret == -1) && (errno == EINTR));
+
+    if (ret == -1) {
+        GGL_LOGE("Failed to truncate fd for write (errno=%d).", errno);
+        return GGL_ERR_FAILURE;
+    }
+    return GGL_ERR_OK;
+}

Diff for: modules/ggl-http/include/ggl/http.h

@@ -98,7 +98,8 @@ GglError sigv4_download(
     GglBuffer host,
     GglBuffer file_path,
     int fd,
-    SigV4Details sigv4_details
+    SigV4Details sigv4_details,
+    long *http_response_code
 );
 
 GglError gg_dataplane_call(

Diff for: modules/ggl-http/src/gghttp.c

@@ -78,7 +78,8 @@ GglError sigv4_download(
     GglBuffer host,
     GglBuffer file_path,
     int fd,
-    SigV4Details sigv4_details
+    SigV4Details sigv4_details,
+    long *http_response_code
 ) {
     CurlData curl_data = { 0 };
     GglError error = gghttplib_init_curl(&curl_data, url_for_sigv4_download);
@@ -153,6 +154,7 @@ GglError sigv4_download(
     long http_status_code = 0;
     curl_easy_getinfo(curl_data.curl, CURLINFO_HTTP_CODE, &http_status_code);
     GGL_LOGD("Return HTTP code: %ld", http_status_code);
+    *http_response_code = http_status_code;
 
     struct curl_header *type = NULL;
     curl_easy_header(

Diff for: modules/ggl-http/src/gghttp_util.c

@@ -9,14 +9,12 @@
 #include <sys/types.h>
 #include <assert.h>
 #include <curl/curl.h>
-#include <errno.h>
 #include <ggl/backoff.h>
 #include <ggl/cleanup.h>
 #include <ggl/file.h>
 #include <ggl/log.h>
 #include <ggl/vector.h>
 #include <pthread.h>
-#include <unistd.h>
 #include <stdbool.h>
 #include <stdlib.h>
 
@@ -109,21 +107,6 @@ static GglError clear_buffer(void *response_data) {
     return GGL_ERR_OK;
 }
 
-static GglError truncate_file(void *response_data) {
-    int fd = *(int *) response_data;
-
-    int ret;
-    do {
-        ret = ftruncate(fd, 0);
-    } while ((ret == -1) && (errno == EINTR));
-
-    if (ret == -1) {
-        GGL_LOGE("Failed to truncate fd for write (errno=%d).", errno);
-        return GGL_ERR_FAILURE;
-    }
-    return GGL_ERR_OK;
-}
-
 static GglError curl_request_retry_wrapper(void *ctx) {
     CurlRequestRetryCtx *retry_ctx = (CurlRequestRetryCtx *) ctx;
     CurlData *curl_data = retry_ctx->curl_data;

Diff for: test_modules/s3-get-test/src/entry.c

@@ -112,20 +112,21 @@ GglError run_s3_test(char *region, char *bucket, char *key, char *file_path) {
         if (request_ret != GGL_ERR_OK) {
             return GGL_ERR_FAILURE;
         }
+        long http_response_code;
 
         request_ret = sigv4_download(
             url_buffer,
             (GglBuffer) { .data = host_vec.buf.data, .len = host_vec.buf.len },
             ggl_buffer_from_null_term(key),
             fd,
-            (SigV4Details) {
-                .aws_region = ggl_buffer_from_null_term(region),
-                .aws_service = GGL_STR("s3"),
-                .access_key_id = aws_access_key_id->buf,
-                .secret_access_key = aws_secret_access_key->buf,
-                .session_token = aws_session_token->buf,
-            }
+            (SigV4Details) { .aws_region = ggl_buffer_from_null_term(region),
+                             .aws_service = GGL_STR("s3"),
+                             .access_key_id = aws_access_key_id->buf,
+                             .secret_access_key = aws_secret_access_key->buf,
+                             .session_token = aws_session_token->buf },
+            &http_response_code
         );
+        (void) http_response_code;
     }
 
     int fd = 0;