Merge pull request #1281 from okorach:Handle-HTTP-errors-in-threaded-search

Gracefully handle HTTP errors in threads

Author: okorach

sonar/issues.py

@@ -23,10 +23,12 @@
 from datetime import date, datetime, timedelta
 import json
 import re
+
 from typing import Union, Optional
 from queue import Queue
 from threading import Thread
 import requests.utils
+from requests.exceptions import HTTPError
 
 import sonar.logging as log
 import sonar.platform as pf
@@ -702,12 +704,15 @@ def __search_thread(queue: Queue) -> None:
         page_params = params.copy()
         page_params["p"] = page
         log.debug("Threaded issue search params = %s", str(page_params))
-        data = json.loads(endpoint.get(api, params=page_params).text)
-        for i in data["issues"]:
-            i["branch"] = page_params.get("branch", None)
-            i["pullRequest"] = page_params.get("pullRequest", None)
-            issue_list[i["key"]] = get_object(endpoint=endpoint, key=i["key"], data=i)
-        log.debug("Added %d issues in threaded search page %d", len(data["issues"]), page)
+        try:
+            data = json.loads(endpoint.get(api, params=page_params).text)
+            for i in data["issues"]:
+                i["branch"] = page_params.get("branch", None)
+                i["pullRequest"] = page_params.get("pullRequest", None)
+                issue_list[i["key"]] = get_object(endpoint=endpoint, key=i["key"], data=i)
+            log.debug("Added %d issues in threaded search page %d", len(data["issues"]), page)
+        except HTTPError as e:
+            log.critical("HTTP Error while searching issues, search may be incomplete: %s", str(e))
         queue.task_done()
 
 

sonar/sqobject.py

@@ -109,12 +109,15 @@ def __search_thread(queue: Queue) -> None:
         page_params = params.copy()
         page_params["p"] = page
         log.debug("Threaded search: API = %s params = %s", api, str(params))
-        data = json.loads(endpoint.get(api, params=page_params).text)
-        for obj in data[returned_field]:
-            if object_class.__name__ in ("QualityProfile", "QualityGate", "Groups", "Portfolio", "Project"):
-                objects[obj[key_field]] = object_class.load(endpoint=endpoint, data=obj)
-            else:
-                objects[obj[key_field]] = object_class(endpoint, obj[key_field], data=obj)
+        try:
+            data = json.loads(endpoint.get(api, params=page_params).text)
+            for obj in data[returned_field]:
+                if object_class.__name__ in ("QualityProfile", "QualityGate", "Groups", "Portfolio", "Project"):
+                    objects[obj[key_field]] = object_class.load(endpoint=endpoint, data=obj)
+                else:
+                    objects[obj[key_field]] = object_class(endpoint, obj[key_field], data=obj)
+        except HTTPError as e:
+            log.critical("HTTP error while searching %s, search skipped: %s", object_class.__name__, str(e))
         queue.task_done()