Retry S3 download on 403 error

Author: rawalexe

modules/ggdeploymentd/src/deployment_handler.c

@@ -14,6 +14,7 @@
 #include <assert.h>
 #include <fcntl.h>
 #include <ggl/alloc.h>
+#include <ggl/backoff.h>
 #include <ggl/base64.h>
 #include <ggl/buffer.h>
 #include <ggl/bump_alloc.h>
@@ -409,6 +410,81 @@ static GglError get_tes_credentials(TesCredentials *tes_creds) {
     return GGL_ERR_OK;
 }
 
+typedef struct CurlRequestRetryCtx {
+    const char *url_for_sigv4_download;
+    GglBuffer host;
+    GglBuffer file_path;
+    SigV4Details sigv4_details;
+
+    // reset response_data for next attempt
+    GglError (*retry_cleanup_fn)(void *);
+    void *response_data;
+
+    // Needed to propagate errors when retrying is impossible.
+    GglError err;
+} DownloadRequestRetryCtx;
+
+static GglError retry_download_wrapper(void *ctx) {
+    DownloadRequestRetryCtx *retry_ctx = (DownloadRequestRetryCtx *) ctx;
+    long http_response_code;
+
+    GglError ret = sigv4_download(
+        retry_ctx->url_for_sigv4_download,
+        retry_ctx->host,
+        retry_ctx->file_path,
+        *(int *) retry_ctx->response_data,
+        retry_ctx->sigv4_details,
+        &http_response_code
+    );
+    if (http_response_code == (long) 403) {
+        GglError err = retry_ctx->retry_cleanup_fn(retry_ctx->response_data);
+        if (err != GGL_ERR_OK) {
+            retry_ctx->err = err;
+            return GGL_ERR_OK;
+        }
+        return GGL_ERR_FAILURE;
+    }
+    if (ret != GGL_ERR_OK) {
+        GGL_LOGE(
+            "Artifact download attempt failed due to error: %d", ret
+
+        );
+        retry_ctx->err = ret;
+        return GGL_ERR_OK;
+    }
+
+    retry_ctx->err = ret;
+    return GGL_ERR_OK;
+}
+
+static GglError retriable_download_request(
+    const char *url_for_sigv4_download,
+    GglBuffer host,
+    GglBuffer file_path,
+    int artifact_fd,
+    SigV4Details sigv4_details
+) {
+    DownloadRequestRetryCtx ctx
+        = { .url_for_sigv4_download = url_for_sigv4_download,
+            .host = host,
+            .file_path = file_path,
+            .sigv4_details = sigv4_details,
+            .response_data = (void *) &artifact_fd,
+            .retry_cleanup_fn = truncate_file,
+            .err = GGL_ERR_OK };
+
+    GglError ret
+        = ggl_backoff(1000, 64000, 3, retry_download_wrapper, (void *) &ctx);
+    if (ret != GGL_ERR_OK) {
+        GGL_LOGE("Artifact download attempt failed; retries exhausted.");
+        return ret;
+    }
+    if (ctx.err != GGL_ERR_OK) {
+        return ctx.err;
+    }
+    return GGL_ERR_OK;
+}
+
 static GglError download_s3_artifact(
     GglBuffer scratch_buffer,
     GglUriInfo uri_info,
@@ -434,7 +510,7 @@ static GglError download_s3_artifact(
         return error;
     }
 
-    return sigv4_download(
+    return retriable_download_request(
         (const char *) url_vec.buf.data,
         (GglBuffer) { .data = &scratch_buffer.data[start_loc],
                       .len = end_loc - start_loc },

modules/ggl-file/include/ggl/file.h

@@ -87,4 +87,6 @@ static inline void cleanup_closedir(DIR **dirp) {
     }
 }
 
+GglError truncate_file(void *response_data);
+
 #endif

modules/ggl-file/src/file.c

@@ -578,3 +578,18 @@ GglError ggl_file_read_path(GglBuffer path, GglBuffer *content) {
 
     return GGL_ERR_OK;
 }
+
+GglError truncate_file(void *response_data) {
+    int fd = *(int *) response_data;
+
+    int ret;
+    do {
+        ret = ftruncate(fd, 0);
+    } while ((ret == -1) && (errno == EINTR));
+
+    if (ret == -1) {
+        GGL_LOGE("Failed to truncate fd for write (errno=%d).", errno);
+        return GGL_ERR_FAILURE;
+    }
+    return GGL_ERR_OK;
+}

modules/ggl-http/include/ggl/http.h

@@ -98,7 +98,8 @@ GglError sigv4_download(
     GglBuffer host,
     GglBuffer file_path,
     int fd,
-    SigV4Details sigv4_details
+    SigV4Details sigv4_details,
+    long *http_response_code
 );
 
 GglError gg_dataplane_call(

modules/ggl-http/src/gghttp.c

@@ -78,7 +78,8 @@ GglError sigv4_download(
     GglBuffer host,
     GglBuffer file_path,
     int fd,
-    SigV4Details sigv4_details
+    SigV4Details sigv4_details,
+    long *http_response_code
 ) {
     CurlData curl_data = { 0 };
     GglError error = gghttplib_init_curl(&curl_data, url_for_sigv4_download);
@@ -153,6 +154,7 @@ GglError sigv4_download(
     long http_status_code = 0;
     curl_easy_getinfo(curl_data.curl, CURLINFO_HTTP_CODE, &http_status_code);
     GGL_LOGD("Return HTTP code: %ld", http_status_code);
+    *http_response_code = http_status_code;
 
     struct curl_header *type = NULL;
     curl_easy_header(

modules/ggl-http/src/gghttp_util.c

@@ -109,21 +109,6 @@ static GglError clear_buffer(void *response_data) {
     return GGL_ERR_OK;
 }
 
-static GglError truncate_file(void *response_data) {
-    int fd = *(int *) response_data;
-
-    int ret;
-    do {
-        ret = ftruncate(fd, 0);
-    } while ((ret == -1) && (errno == EINTR));
-
-    if (ret == -1) {
-        GGL_LOGE("Failed to truncate fd for write (errno=%d).", errno);
-        return GGL_ERR_FAILURE;
-    }
-    return GGL_ERR_OK;
-}
-
 static GglError curl_request_retry_wrapper(void *ctx) {
     CurlRequestRetryCtx *retry_ctx = (CurlRequestRetryCtx *) ctx;
     CurlData *curl_data = retry_ctx->curl_data;