Fix sonar-audit bugs (#1412)

* Fixes #1410

* Align requests error on ERROR

* Fixes #1411

* Remove crlf in error log

* Add logging on objects search

* Speedup QP loading

Author: okorach

cli/findings_export.py

@@ -324,7 +324,7 @@ def __get_component_findings(queue: Queue[tuple[object, ConfigSettings]], write_
                 try:
                     findings_list = component.get_issues(filters=new_params)
                 except (ConnectionError, RequestException) as e:
-                    log.critical("%s while exporting issues of %s, skipped", util.error_msg(e), str(component))
+                    log.error("%s while exporting issues of %s, skipped", util.error_msg(e), str(component))
                     findings_list = {}
             else:
                 log.debug("Status = %s, Types = %s, Resol = %s, Sev = %s", str(i_statuses), str(i_types), str(i_resols), str(i_sevs))

sonar/permissions/permission_templates.py

@@ -180,7 +180,7 @@ def to_json(self, export_settings: types.ConfigSettings = None) -> types.ObjectJ
         return utilities.remove_nones(utilities.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False)))
 
     def _audit_pattern(self, audit_settings: types.ConfigSettings) -> list[pb.Problem]:
-        log.debug("Auditing %s projectKeyPattern ('%s')", str(self.project_key_pattern))
+        log.debug("Auditing %s projectKeyPattern ('%s')", str(self), str(self.project_key_pattern))
         if not self.project_key_pattern or self.project_key_pattern == "":
             if not (self.is_applications_default() or self.is_portfolios_default() or self.is_projects_default()):
                 return [pb.Problem(get_rule(RuleId.TEMPLATE_WITH_NO_PATTERN), self, str(self))]

sonar/qualityprofiles.py

@@ -242,7 +242,8 @@ def rules(self) -> dict[str, rules.Rule]:
         if self._rules is not None:
             # Assume nobody changed QP during execution
             return self._rules
-        self._rules = rules.search(self.endpoint, activation="true", qprofile=self.key, s="key", languages=self.language)
+        rule_key_list = rules.search_keys(self.endpoint, activation="true", qprofile=self.key, s="key", languages=self.language)
+        self._rules = {k: rules.get_object(self.endpoint, k) for k in rule_key_list}
         return self._rules
 
     def activate_rule(self, rule_key: str, severity: str = None, **params) -> bool:
@@ -534,10 +535,10 @@ def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings = None) ->
     :rtype: list[Problem]
     """
     log.info("--- Auditing quality profiles ---")
-    get_list(endpoint=endpoint)
+    rules.get_list(endpoint=endpoint)
     problems = []
     langs = {}
-    for qp in search(endpoint).values():
+    for qp in search(endpoint=endpoint).values():
         problems += qp.audit(audit_settings)
         langs[qp.language] = langs.get(qp.language, 0) + 1
     for lang, nb_qp in langs.items():

sonar/rules.py

@@ -244,10 +244,27 @@ def get_facet(facet: str, endpoint: platform.Platform) -> dict[str, str]:
 
 
 def search(endpoint: platform.Platform, **params) -> dict[str, Rule]:
-    """Searches ruless with optional filters"""
+    """Searches rules with optional filters"""
     return sq.search_objects(endpoint=endpoint, object_class=Rule, params=params, threads=4)
 
 
+def search_keys(endpoint: platform.Platform, **params) -> list[str]:
+    """Searches ruless with optional filters"""
+    new_params = params.copy() if params else {}
+    new_params["ps"] = 500
+    new_params["p"], nbr_pages = 0, 1
+    rule_list = []
+    try:
+        while new_params["p"] < nbr_pages:
+            new_params["p"] += 1
+            data = json.loads(endpoint.get(Rule.SEARCH_API, params=new_params).text)
+            nbr_pages = utilities.nbr_pages(data)
+            rule_list += [r[Rule.SEARCH_KEY_FIELD] for r in data[Rule.SEARCH_RETURN_FIELD]]
+    except (ConnectionError, RequestException) as e:
+        log.error("%s while getting rules", utilities.error_msg(e))
+    return rule_list
+
+
 def count(endpoint: platform.Platform, **params) -> int:
     """Count number of rules that correspond to certain filters"""
     return json.loads(endpoint.get(Rule.SEARCH_API, params={**params, "ps": 1}).text)["total"]

sonar/sqobject.py

@@ -133,12 +133,14 @@ def search_objects(endpoint: object, object_class: any, params: types.ApiParams,
     new_params["p"] = 1
     objects_list = {}
     data = json.loads(endpoint.get(api, params=new_params).text)
+    nb_pages = utilities.nbr_pages(data)
+    nb_objects = max(len(data[returned_field]), utilities.nbr_total_elements(data))
+    log.info("%d %s to load", nb_objects, object_class.__name__)
     for obj in data[returned_field]:
         if object_class.__name__ in ("Portfolio", "Group", "QualityProfile", "User", "Application", "Project", "Organization"):
             objects_list[obj[key_field]] = object_class.load(endpoint=endpoint, data=obj)
         else:
             objects_list[obj[key_field]] = object_class(endpoint, obj[key_field], data=obj)
-    nb_pages = utilities.nbr_pages(data)
     if nb_pages == 1:
         # If everything is returned on the 1st page, no multi-threading needed
         return objects_list

sonar/tasks.py

@@ -24,6 +24,8 @@
 import json
 import re
 
+from requests import RequestException
+
 import sonar.logging as log
 import sonar.sqobject as sq
 import sonar.platform as pf
@@ -530,8 +532,12 @@ def search(endpoint: pf.Platform, only_current: bool = False, component_key: str
         params["onlyCurrents"] = "true"
     if component_key is not None:
         params["component"] = component_key
-    data = json.loads(endpoint.get("ce/activity", params=params).text)
-    return [Task(endpoint=endpoint, task_id=t["id"], data=t) for t in data["tasks"]]
+    try:
+        data = json.loads(endpoint.get("ce/activity", params=params).text)
+        return [Task(endpoint=endpoint, task_id=t["id"], data=t) for t in data["tasks"]]
+    except (ConnectionError, RequestException) as e:
+        log.error("%s while getting background tasks of component '%s'", util.error_msg(e), str(component_key))
+    return []
 
 
 def search_all_last(endpoint: pf.Platform) -> list[Task]:

sonar/utilities.py

@@ -440,7 +440,7 @@ def http_error_and_code(exception: requests.HTTPError) -> tuple[int, str]:
     response = exception.response
     if response.ok:
         return None, "No error"
-    tool_msg = f"For request URL {response.request.url}\n"
+    tool_msg = f"URL {response.request.url}: "
     code = response.status_code
     if code == HTTPStatus.UNAUTHORIZED:
         tool_msg += f"HTTP error {code} - Authentication error. Is token valid ?"