fix: OIDC token refresh recovery after network disruption

CONTRIBUTORS

@@ -2,3 +2,4 @@
 Nic Fragale <nic.fragale@netfoundry.io>
 Tom Carroll <Thomas.Carroll@pnnl.gov>
 Mario Trangoni <mario.trangoni@gmail.com>
+Dominik Münsterer <dominik.muensterer@deltasecure.de>

inc_internal/oidc.h

@@ -60,6 +60,9 @@ struct oidc_client_s {
     bool need_refresh;
     struct auth_req *request;
     tlsuv_http_req_t *refresh_req;
+    int refresh_failures;
+    int total_refresh_failures;
+    uint64_t token_expiry;
 };
 
 // init

library/oidc.c

@@ -102,7 +102,9 @@ int oidc_client_init(uv_loop_t *loop, oidc_client_t *clt,
         return rc;
     }
     tlsuv_http_set_ssl(&clt->http, tls);
+    tlsuv_http_connect_timeout(&clt->http, 15000);
 
+    clt->token_expiry = 0;
     clt->timer = calloc(1, sizeof(*clt->timer));
     uv_timer_init(loop, clt->timer);
     clt->timer->data = clt;
@@ -570,12 +572,8 @@ static void oidc_client_set_tokens(oidc_client_t *clt, json_object *tok_json) {
     }
     if (ttl) {
         int32_t t = json_object_get_int(ttl);
-        if (t <= 60) {
-            OIDC_LOG(WARN, "token lifetime is too short[%d seconds]. this may cause problems", t);
-            t = t / 2;
-        } else {
-            t = t - 30; // refresh 30 seconds before expiry
-        }
+        clt->token_expiry = uv_now(clt->timer->loop) + (uint64_t)t * 1000;
+        t = t / 2; // refresh at half the token lifetime
         OIDC_LOG(DEBUG, "scheduling token refresh in %d seconds", t);
         uv_timer_start(clt->timer, refresh_time_cb, t * 1000, 0);
     }
@@ -588,33 +586,75 @@ static void refresh_cb(tlsuv_http_resp_t *http_resp, const char *err, json_objec
 
     if (http_resp->code == 200 && resp != NULL) {
         OIDC_LOG(DEBUG,  "token refresh success");
+        clt->refresh_failures = 0;
+        clt->total_refresh_failures = 0;
         oidc_client_set_tokens(clt, resp);
         return;
     }
 
-    if (http_resp->code >= 0 || http_resp->code == UV_EOF) {
-        // controller may abruptly terminate shutdown connection (EOF) if auth has failed
-        OIDC_LOG(WARN, "OIDC token refresh failed: %d %s [%s]",
-                 http_resp->code, http_resp->status, err);
-        if (resp) {
-            OIDC_LOG(WARN, "response: %s", json_object_get_string(resp));
-        }
-        json_object_put(clt->tokens);
-        clt->tokens = NULL;
-
-        oidc_client_start(clt, clt->token_cb);
-    }
-
     if (http_resp->code == UV_ECANCELED) {
         OIDC_LOG(DEBUG, "OIDC token refresh was canceled");
         return;
     }
 
-    if (http_resp->code < 0) {  // connection failure, try another refresh
-        OIDC_LOG(WARN, "OIDC token refresh failed (trying again): %d/%s", http_resp->code, err);
-        uv_timer_start(clt->timer, refresh_time_cb, 5 * 1000, 0);
+    if (http_resp->code < 0) {
+        clt->refresh_failures++;
+        clt->total_refresh_failures++;
+        OIDC_LOG(WARN, "OIDC token refresh failed (%d/%s), attempt %d (total: %d)",
+                 http_resp->code, err, clt->refresh_failures, clt->total_refresh_failures);
+
+        // token expired, give up on refresh and restart full auth
+        uint64_t now = uv_now(clt->timer->loop);
+        if (clt->token_expiry > 0 && now >= clt->token_expiry) {
+            OIDC_LOG(WARN, "OIDC token has expired after %d refresh attempts, restarting full authentication",
+                     clt->total_refresh_failures);
+            tlsuv_http_cancel_all(&clt->http);
+            clt->refresh_failures = 0;
+            clt->total_refresh_failures = 0;
+            clt->token_expiry = 0;
+            json_object_put(clt->tokens);
+            clt->tokens = NULL;
+            oidc_client_start(clt, clt->token_cb);
+            return;
+        }
+
+        if (clt->refresh_failures >= 3) {
+            OIDC_LOG(WARN, "resetting OIDC connection after %d consecutive failures", clt->refresh_failures);
+            tlsuv_http_cancel_all(&clt->http);
+            clt->refresh_failures = 0;
+        }
+
+        // exponential backoff with jitter to avoid stampeding controller after outage
+        int shift = clt->total_refresh_failures - 1;
+        if (shift > 4) shift = 4; // cap at 80s base
+        uint64_t delay = 5000ULL << shift;
+        if (delay > 60000) delay = 60000;
+        // jitter: [delay/2, delay]
+        uint32_t half = (uint32_t)(delay / 2);
+        if (half > 0) delay = half + randombytes_uniform(half + 1);
+        // clamp to remaining token lifetime
+        if (clt->token_expiry > 0 && now + delay > clt->token_expiry) {
+            delay = clt->token_expiry - now;
+        }
+
+        OIDC_LOG(DEBUG, "scheduling token refresh retry in %llu ms", (unsigned long long)delay);
+        uv_timer_start(clt->timer, refresh_time_cb, delay, 0);
         return;
     }
+
+    // http_resp->code > 0 but not 200: server-side rejection (e.g. 401, 403, etc.)
+    OIDC_LOG(WARN, "OIDC token refresh failed: %d %s [%s]",
+             http_resp->code, http_resp->status, err);
+    if (resp) {
+        OIDC_LOG(WARN, "response: %s", json_object_get_string(resp));
+    }
+    clt->refresh_failures = 0;
+    clt->total_refresh_failures = 0;
+    clt->token_expiry = 0;
+    json_object_put(clt->tokens);
+    clt->tokens = NULL;
+
+    oidc_client_start(clt, clt->token_cb);
 }
 
 static const char* get_basic_auth_header(const char *client_id) {