Fix issues search thread exhaustion (#1396)

* Log all HTTP errors

* Cleaner project export progress log status

* Catch RequestException and ConnectionError everywhere

* Change http_error in error_msg

* Quality pass

* Fix redundant exception

* Remove unused import HTTPError

* Quality pass

Author: okorach

cli/findings_export.py

@@ -32,7 +32,7 @@
 from queue import Queue
 import threading
 from threading import Thread
-from requests.exceptions import HTTPError
+from requests import HTTPError, RequestException
 
 from cli import options
 from sonar.util.types import ConfigSettings
@@ -294,8 +294,8 @@ def __get_component_findings(queue: Queue[tuple[object, ConfigSettings]], write_
                 findings_list = findings.export_findings(
                     component.endpoint, component.key, branch=params.get("branch", None), pull_request=params.get("pullRequest", None)
                 )
-            except HTTPError as e:
-                log.critical("Error %s while exporting findings of %s, skipped", str(e), str(component))
+            except (ConnectionError, RequestException) as e:
+                log.critical("%s while exporting findings of %s, skipped", util.error_msg(e), str(component))
                 findings_list = {}
             write_queue.put([findings_list, False])
         else:
@@ -323,8 +323,8 @@ def __get_component_findings(queue: Queue[tuple[object, ConfigSettings]], write_
             if (i_statuses or not status_list) and (i_resols or not resol_list) and (i_types or not type_list) and (i_sevs or not sev_list):
                 try:
                     findings_list = component.get_issues(filters=new_params)
-                except HTTPError as e:
-                    log.critical("Error %s while exporting issues of %s, skipped", str(e), str(component))
+                except (ConnectionError, RequestException) as e:
+                    log.critical("%s while exporting issues of %s, skipped", util.error_msg(e), str(component))
                     findings_list = {}
             else:
                 log.debug("Status = %s, Types = %s, Resol = %s, Sev = %s", str(i_statuses), str(i_types), str(i_resols), str(i_sevs))
@@ -333,8 +333,8 @@ def __get_component_findings(queue: Queue[tuple[object, ConfigSettings]], write_
             if (h_statuses or not status_list) and (h_resols or not resol_list) and (h_types or not type_list) and (h_sevs or not sev_list):
                 try:
                     findings_list.update(component.get_hotspots(filters=new_params))
-                except HTTPError as e:
-                    log.critical("Error %s while exporting hotspots of object key %s, skipped", str(e), str(component))
+                except (ConnectionError, RequestException) as e:
+                    log.error("%s while exporting hotspots of object key %s, skipped", util.error_msg(e), str(component))
             else:
                 log.debug("Status = %s, Types = %s, Resol = %s, Sev = %s", str(h_statuses), str(h_types), str(h_resols), str(h_sevs))
                 log.info("Selected types, severities, resolutions or statuses disables issue search")
@@ -351,8 +351,8 @@ def store_findings(components_list: dict[str, object], params: ConfigSettings) -
         try:
             log.debug("Queue %s task %s put", str(my_queue), str(comp))
             my_queue.put((comp, params.copy()))
-        except HTTPError as e:
-            log.critical("Error %s while exporting findings of %s, skipped", str(e), str(comp))
+        except (ConnectionError, RequestException) as e:
+            log.critical("%s while exporting findings of %s, skipped", util.error_msg(e), str(comp))
 
     threads = params.get(options.NBR_THREADS, 4)
     for i in range(min(threads, len(components_list))):

cli/loc.py

@@ -24,7 +24,7 @@
 import sys
 import csv
 import datetime
-from requests.exceptions import HTTPError
+from requests import RequestException
 
 from cli import options
 import sonar.logging as log
@@ -51,8 +51,8 @@ def __get_csv_row(o: object, **kwargs) -> tuple[list[str], str]:
     """Returns CSV row of object"""
     try:
         loc = o.loc()
-    except HTTPError as e:
-        log.warning("HTTP Error %s, LoC export of %s skipped", str(e), str(o))
+    except (ConnectionError, RequestException) as e:
+        log.warning("%s, LoC export of %s skipped", util.error_msg(e), str(o))
         loc = ""
     arr = [o.key, loc]
     obj_type = type(o).__name__.lower()
@@ -113,8 +113,8 @@ def __get_object_json_data(o: object, **kwargs) -> dict[str, str]:
         d = {parent_type: o.concerned_object.key, "branch": o.name, "ncloc": ""}
     try:
         d["ncloc"] = o.loc()
-    except HTTPError as e:
-        log.warning("HTTP Error %s, LoC export of %s skipped", str(e), str(o))
+    except (ConnectionError, RequestException) as e:
+        log.warning("%s, LoC export of %s skipped", util.error_msg(e), str(o))
     if kwargs[options.WITH_NAME]:
         d[f"{parent_type}Name"] = o.name
         if obj_type in ("branch", "applicationbranch"):

cli/measures_export.py

@@ -29,8 +29,7 @@
 
 from typing import Union
 
-from http import HTTPStatus
-from requests.exceptions import HTTPError
+from requests import RequestException
 from sonar.util import types
 from cli import options
 import sonar.logging as log
@@ -56,12 +55,7 @@ def __last_analysis(component: object) -> str:
 
 def __get_json_measures_history(obj: object, wanted_metrics: types.KeyList) -> dict[str, str]:
     """Returns the measure history of an object (project, branch, application, portfolio)"""
-    data = {}
-    try:
-        data["history"] = obj.get_measures_history(wanted_metrics)
-    except HTTPError as e:
-        log.error("HTTP Error %s, measures history export of %s skipped", str(e), str(obj))
-    return data
+    return {"history": obj.get_measures_history(wanted_metrics)}
 
 
 def __get_object_measures(obj: object, wanted_metrics: types.KeyList) -> dict[str, str]:
@@ -288,11 +282,8 @@ def __get_measures(obj: object, wanted_metrics: types.KeyList, hist: bool) -> Un
             data.update(__get_json_measures_history(obj, wanted_metrics))
         else:
             data.update(__get_object_measures(obj, wanted_metrics))
-    except HTTPError as e:
-        if e.response.status_code == HTTPStatus.FORBIDDEN:
-            log.error("Insufficient permission to retrieve measures of %s, export skipped for this object", str(obj))
-        else:
-            log.error("HTTP Error %s while retrieving measures of %s, export skipped for this object", str(e), str(obj))
+    except (ConnectionError, RequestException) as e:
+        log.error("%s, measures export skipped for %s", util.error_msg(e), str(obj))
         return None
     return data
 

sonar/app_branches.py

@@ -24,7 +24,7 @@
 
 import json
 from http import HTTPStatus
-from requests.exceptions import HTTPError
+from requests import RequestException
 from requests.utils import quote
 
 import sonar.logging as log
@@ -34,7 +34,7 @@
 
 from sonar.applications import Application as App
 from sonar.branches import Branch
-from sonar import exceptions, projects
+from sonar import exceptions, projects, utilities
 import sonar.sqobject as sq
 
 _OBJECTS = {}
@@ -109,9 +109,11 @@ def create(cls, app: App, name: str, project_branches: list[Branch]) -> Applicat
             params["projectBranch"].append(br_name)
         try:
             app.endpoint.post(APIS["create"], params=params)
-        except HTTPError as e:
+        except (ConnectionError, RequestException) as e:
             if e.response.status_code == HTTPStatus.BAD_REQUEST:
                 raise exceptions.ObjectAlreadyExists(f"app.App {app.key} branch '{name}", e.response.text)
+            log.critical("%s while creating branch '%s' of '%s'", utilities.error_msg(e), name, str(app))
+            raise
         return ApplicationBranch(app=app, name=name, project_branches=project_branches)
 
     @classmethod
@@ -182,9 +184,12 @@ def update(self, name: str, project_branches: list[Branch]) -> bool:
             params["projectBranch"].append(br_name)
         try:
             ok = self.endpoint.post(APIS["update"], params=params).ok
-        except HTTPError as e:
+        except (ConnectionError, RequestException) as e:
             if e.response.status_code == HTTPStatus.NOT_FOUND:
                 raise exceptions.ObjectNotFound(str(self), e.response.text)
+            log.error("%s while updating '%s'", utilities.error_msg(e), str(self))
+            raise
+
         self.name = name
         self._project_branches = project_branches
         return ok

sonar/applications.py

@@ -26,7 +26,7 @@
 from datetime import datetime
 from http import HTTPStatus
 from threading import Lock
-from requests.exceptions import HTTPError
+from requests import RequestException
 
 import sonar.logging as log
 import sonar.platform as pf
@@ -90,9 +90,11 @@ def get_object(cls, endpoint: pf.Platform, key: str) -> Application:
             return _OBJECTS[uu]
         try:
             data = json.loads(endpoint.get(APIS["get"], params={"application": key}).text)["application"]
-        except HTTPError as e:
+        except (ConnectionError, RequestException) as e:
             if e.response.status_code == HTTPStatus.NOT_FOUND:
                 raise exceptions.ObjectNotFound(key, f"Application key '{key}' not found")
+            log.error("%s while getting app key '%s'", util.error_msg(e), key)
+            raise
         return cls.load(endpoint, data)
 
     @classmethod
@@ -131,9 +133,11 @@ def create(cls, endpoint: pf.Platform, key: str, name: str) -> Application:
         check_supported(endpoint)
         try:
             endpoint.post(APIS["create"], params={"key": key, "name": name})
-        except HTTPError as e:
+        except (ConnectionError, RequestException) as e:
             if e.response.status_code == HTTPStatus.BAD_REQUEST:
                 raise exceptions.ObjectAlreadyExists(key, e.response.text)
+            log.critical("%s while creating app key '%s'", util.error_msg(e), key)
+            raise
         return Application(endpoint, key, name)
 
     def refresh(self) -> None:
@@ -146,10 +150,11 @@ def refresh(self) -> None:
         try:
             self.reload(json.loads(self.get("navigation/component", params={"component": self.key}).text))
             self.reload(json.loads(self.get(APIS["get"], params=self.search_params()).text)["application"])
-        except HTTPError as e:
+        except (ConnectionError, RequestException) as e:
             if e.response.status_code == HTTPStatus.NOT_FOUND:
                 _OBJECTS.pop(self.uuid(), None)
                 raise exceptions.ObjectNotFound(self.key, f"{str(self)} not found")
+            log.error("%s while refreshing %s", util.error_msg(e), str(self))
             raise
 
     def __str__(self) -> str:
@@ -383,11 +388,12 @@ def add_projects(self, project_list: list[str]) -> bool:
             try:
                 r = self.post("applications/add_project", params={"application": self.key, "project": proj})
                 ok = ok and r.ok
-            except HTTPError as e:
+            except (ConnectionError, RequestException) as e:
                 if e.response.status_code == HTTPStatus.NOT_FOUND:
                     log.warning("Project '%s' not found, can't be added to %s", proj, self)
                     ok = False
                 else:
+                    log.error("%s while adding projects to %s", util.error_msg(e), str(self))
                     raise
         return ok
 

sonar/branches.py

@@ -22,7 +22,7 @@
 from http import HTTPStatus
 import json
 from urllib.parse import unquote
-from requests.exceptions import HTTPError
+from requests import HTTPError, RequestException
 import requests.utils
 
 from sonar import platform
@@ -89,9 +89,11 @@ def get_object(cls, concerned_object: projects.Project, branch_name: str) -> Bra
             return _OBJECTS[uu]
         try:
             data = json.loads(concerned_object.endpoint.get(APIS["list"], params={"project": concerned_object.key}).text)
-        except HTTPError as e:
-            if e.response.status_code == HTTPStatus.NOT_FOUND:
+        except (ConnectionError, RequestException) as e:
+            if isinstance(HTTPError, e) and e.response.status_code == HTTPStatus.NOT_FOUND:
                 raise exceptions.ObjectNotFound(concerned_object.key, f"Project '{concerned_object.key}' not found")
+            log.critical("%s while getting branch '%s' of %s", util.error_msg(e), branch_name, str(concerned_object))
+            raise
         for br in data.get("branches", []):
             if br["name"] == branch_name:
                 return cls.load(concerned_object, branch_name, br)
@@ -127,9 +129,10 @@ def refresh(self) -> Branch:
         """
         try:
             data = json.loads(self.get(APIS["list"], params={"project": self.concerned_object.key}).text)
-        except HTTPError as e:
-            if e.response.status_code == HTTPStatus.NOT_FOUND:
+        except (ConnectionError, RequestException) as e:
+            if isinstance(HTTPError, e) and e.response.status_code == HTTPStatus.NOT_FOUND:
                 raise exceptions.ObjectNotFound(self.key, f"{str(self)} not found in SonarQube")
+            log.error("%s while refreshing %s", util.error_msg(e), str(self))
         for br in data.get("branches", []):
             if br["name"] == self.name:
                 self._load(br)
@@ -173,7 +176,7 @@ def is_main(self):
             self.refresh()
         return self._is_main
 
-    def delete(self):
+    def delete(self) -> bool:
         """Deletes a branch
 
         :raises ObjectNotFound: Branch not found for deletion
@@ -182,9 +185,11 @@ def delete(self):
         """
         try:
             return sq.delete_object(self, APIS["delete"], {"branch": self.name, "project": self.concerned_object.key}, _OBJECTS)
-        except HTTPError as e:
-            if e.response.status_code == HTTPStatus.BAD_REQUEST:
+        except (ConnectionError, RequestException) as e:
+            if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.BAD_REQUEST:
                 log.warning("Can't delete %s, it's the main branch", str(self))
+            else:
+                log.error("%s while deleting %s", util.error_msg(e), str(self))
             return False
 
     def new_code(self) -> str:
@@ -197,11 +202,10 @@ def new_code(self) -> str:
         elif self._new_code is None:
             try:
                 data = json.loads(self.get(api=APIS["get_new_code"], params={"project": self.concerned_object.key}).text)
-            except HTTPError as e:
-                if e.response.status_code == HTTPStatus.NOT_FOUND:
+            except (ConnectionError, RequestException) as e:
+                if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.NOT_FOUND:
                     raise exceptions.ObjectNotFound(self.concerned_object.key, f"{str(self.concerned_object)} not found")
-                if e.response.status_code == HTTPStatus.FORBIDDEN:
-                    log.error("Error 403 when getting new code period of %s", {str(self)})
+                log.error("%s while getting new code period of %s", util.error_msg(e), str(self))
                 raise e
             for b in data["newCodePeriods"]:
                 new_code = settings.new_code_to_string(b)
@@ -261,9 +265,11 @@ def rename(self, new_name):
         log.info("Renaming main branch of %s from '%s' to '%s'", str(self.concerned_object), self.name, new_name)
         try:
             self.post(APIS["rename"], params={"project": self.concerned_object.key, "name": new_name})
-        except HTTPError as e:
-            if e.response.status_code == HTTPStatus.NOT_FOUND:
+        except (ConnectionError, RequestException) as e:
+            if isinstance(HTTPError, e) and e.response.status_code == HTTPStatus.NOT_FOUND:
                 raise exceptions.ObjectNotFound(self.concerned_object.key, f"str{self.concerned_object} not found")
+            log.error("%s while renaming %s", util.error_msg(e), str(self))
+            raise
         _OBJECTS.pop(self.uuid(), None)
         self.name = new_name
         _OBJECTS[self.uuid()] = self
@@ -358,11 +364,8 @@ def audit(self, audit_settings: types.ConfigSettings) -> list[Problem]:
             log.debug("Auditing %s", str(self))
             try:
                 return self.__audit_last_analysis(audit_settings) + self.__audit_zero_loc() + self.__audit_never_analyzed()
-            except HTTPError as e:
-                if e.response.status_code == HTTPStatus.FORBIDDEN:
-                    log.error("Not enough permission to fully audit %s", str(self))
-                else:
-                    log.error("HTTP error %s while auditing %s", str(e), str(self))
+            except Exception as e:
+                log.error("%s while auditing %s, audit skipped", util.error_msg(e), str(self))
         else:
             log.debug("Branch audit disabled, skipping audit of %s", str(self))
         return []

sonar/devops.py

@@ -23,7 +23,7 @@
 from http import HTTPStatus
 import json
 
-from requests.exceptions import HTTPError
+from requests import RequestException
 
 import sonar.logging as log
 from sonar.util import types
@@ -109,10 +109,11 @@ def create(cls, endpoint: platform.Platform, key: str, plt_type: str, url_or_wor
             elif plt_type == "bitbucketcloud":
                 params.update({"clientSecret": _TO_BE_SET, "clientId": _TO_BE_SET, "workspace": url_or_workspace})
                 endpoint.post(_CREATE_API_BBCLOUD, params=params)
-        except HTTPError as e:
+        except (ConnectionError, RequestException) as e:
             if e.response.status_code == HTTPStatus.BAD_REQUEST and endpoint.edition() in ("community", "developer"):
                 log.warning("Can't set DevOps platform '%s', don't you have more that 1 of that type?", key)
                 raise exceptions.UnsupportedOperation(f"Can't set DevOps platform '{key}', don't you have more that 1 of that type?")
+            log.error("%s while creating devops platform %s/%s/%s", util.error_msg(e), key, plt_type, url_or_workspace)
             raise
         o = DevopsPlatform(endpoint=endpoint, key=key, platform_type=plt_type)
         o.refresh()

sonar/hotspots.py

@@ -24,7 +24,7 @@
 import json
 import re
 from http import HTTPStatus
-from requests.exceptions import HTTPError
+from requests import RequestException
 import requests.utils
 
 import sonar.logging as log
@@ -402,12 +402,13 @@ def search(endpoint: pf.Platform, filters: types.ApiParams = None) -> dict[str,
             try:
                 data = json.loads(endpoint.get(Hotspot.SEARCH_API, params=inline_filters, mute=(HTTPStatus.NOT_FOUND,)).text)
                 nbr_hotspots = util.nbr_total_elements(data)
-            except HTTPError as e:
+            except (ConnectionError, RequestException) as e:
                 if e.response.status_code == HTTPStatus.NOT_FOUND:
                     log.warning("No hotspots found with search params %s", str(inline_filters))
                     nbr_hotspots = 0
                     return {}
-                raise e
+                log.error("%s while searching hotspots", util.error_msg(e))
+                break
             nbr_pages = util.nbr_pages(data)
             log.debug("Number of hotspots: %d - Page: %d/%d", nbr_hotspots, inline_filters["p"], nbr_pages)
             if nbr_hotspots > Hotspot.MAX_SEARCH:
@@ -494,8 +495,10 @@ def post_search_filter(hotspots_dict: dict[str, Hotspot], filters: types.ApiPara
             lang = rules.get_object(endpoint=finding.endpoint, key=finding.rule).language
             if lang not in filters["languages"]:
                 filtered_findings.pop(key, None)
+        # pylint: disable-next=E0606
         if "createdAfter" in filters and finding.creation_date < min_date:
             filtered_findings.pop(key, None)
+        # pylint: disable-next=E0606
         if "createdBefore" in filters and finding.creation_date > max_date:
             filtered_findings.pop(key, None)