[SAC-27535] Updated http.py to improve rate limit handling

Author: gl-romil

tap_frontapp/http.py

@@ -1,39 +1,40 @@
 import json
 import time
-
 import requests
 import backoff
 import singer
 from singer import metrics
 
 LOGGER = singer.get_logger()
 
-
 class RateLimitException(Exception):
     pass
 
-
 class MetricsRateLimitException(Exception):
     pass
 
-
 class Client(object):
     BASE_URL = 'https://api2.frontapp.com'
 
     def __init__(self, config):
         self.token = 'Bearer ' + config.get('token')
         self.session = requests.Session()
-
         self.calls_remaining = None
         self.limit_reset = None
 
     def url(self, path):
         return self.BASE_URL + path
 
+    # Log backoff retries
+    def log_backoff(details):
+        LOGGER.warning(f"[Backoff] Retrying {details['target'].__name__} in {details['wait']}s due to {details['exception']}")
+
+    # Retry strategy: Max 5 retries or 60 seconds total
     @backoff.on_exception(backoff.expo,
                           RateLimitException,
-                          max_tries=10,
-                          factor=2)
+                          max_tries=5,
+                          max_time=60,
+                          on_backoff=log_backoff)
     def request(self, method, url, **kwargs):
         if self.calls_remaining is not None and self.calls_remaining == 0:
             wait = self.limit_reset - int(time.monotonic())
@@ -53,13 +54,12 @@ def request(self, method, url, **kwargs):
             with metrics.http_request_timer(endpoint) as timer:
                 response = requests.request(method, url, **kwargs)
                 timer.tags[metrics.Tag.http_status_code] = response.status_code
-
-
         else:
             response = requests.request(method, url, **kwargs)
 
-        self.calls_remaining = int(response.headers['X-Ratelimit-Remaining'])
-        self.limit_reset = int(float(response.headers['X-Ratelimit-Reset']))
+        #Handled missing headers to avoid KeyError
+        self.calls_remaining = int(response.headers.get('X-Ratelimit-Remaining', 1))
+        self.limit_reset = int(float(response.headers.get('X-Ratelimit-Reset', time.monotonic() + 1)))
 
         if response.status_code in [429, 503]:
             raise RateLimitException(response.text)
@@ -83,10 +83,9 @@ def create_report(self, path, data, **kwargs):
         response = self.request('post', url, **kwargs)
         if response.json().get('_links', {}).get('self'):
             return response.json()['_links']['self']
-
         return {}
 
     def list_metrics(self, path, **kwargs):
         url = self.url(path)
         response = self.request('get', url, **kwargs)
-        return response.json().get('_results', [])
+        return response.json().get('_results', [])