Stream sonar-audit output

okorach · web-flow · commit ca4eb8ab390e · 2024-10-13T21:26:35.000+02:00
Fixes #1395 (#1419)
diff --git a/cli/audit.py b/cli/audit.py
@@ -25,10 +25,13 @@
 """
 import sys
 import json
+import csv
+from typing import TextIO
+from threading import Thread, Lock
+from queue import Queue
 
 from cli import options
 
-from sonar.util.types import ConfigSettings
 from sonar import errcodes, exceptions
 from sonar.util import types
 import sonar.logging as log
@@ -47,6 +50,8 @@
     options.WHAT_PORTFOLIOS,
 ]
 
+_WRITE_LOCK = Lock()
+
 
 def _audit_sif(sysinfo, audit_settings):
     log.info("Auditing SIF file '%s'", sysinfo)
@@ -67,40 +72,86 @@ def _audit_sif(sysinfo, audit_settings):
     return (server_id, sif_obj.audit(audit_settings))
 
 
+def write_problems(queue: Queue[list[problem.Problem]], fd: TextIO, settings: types.ConfigSettings) -> None:
+    """
+    Thread to write problems in a CSV file
+    """
+    csvwriter = csv.writer(fd, delimiter=settings.get("CSV_DELIMITER", ","))
+    server_id = settings.get("SERVER_ID", None)
+    with_url = settings.get("WITH_URL", False)
+    while True:
+        problems = queue.get()
+        if problems is None:
+            queue.task_done()
+            break
+        for p in problems:
+            data = []
+            if server_id is not None:
+                data = [server_id]
+            data += list(p.to_json(with_url).values())
+            csvwriter.writerow(data)
+        queue.task_done()
+    log.info("Writing audit probelms complete")
+
+
 def _audit_sq(
-    sq: platform.Platform, settings: ConfigSettings, what_to_audit: list[str] = None, key_list: types.KeyList = None
+    sq: platform.Platform, settings: types.ConfigSettings, what_to_audit: list[str] = None, key_list: types.KeyList = None
 ) -> list[problem.Problem]:
     """Audits a SonarQube/Cloud platform"""
     problems = []
+    q = Queue(maxsize=0)
     everything = False
     if not what_to_audit:
         everything = True
         what_to_audit = options.WHAT_AUDITABLE
-    if options.WHAT_PROJECTS in what_to_audit:
-        problems += projects.audit(endpoint=sq, audit_settings=settings, key_list=key_list)
-    if options.WHAT_PROFILES in what_to_audit:
-        problems += qualityprofiles.audit(endpoint=sq, audit_settings=settings)
-    if options.WHAT_GATES in what_to_audit:
-        problems += qualitygates.audit(endpoint=sq, audit_settings=settings)
-    if options.WHAT_SETTINGS in what_to_audit:
-        problems += sq.audit(audit_settings=settings)
-    if options.WHAT_USERS in what_to_audit:
-        problems += users.audit(endpoint=sq, audit_settings=settings)
-    if options.WHAT_GROUPS in what_to_audit:
-        problems += groups.audit(endpoint=sq, audit_settings=settings)
-    if options.WHAT_PORTFOLIOS in what_to_audit:
-        try:
-            problems += portfolios.audit(endpoint=sq, audit_settings=settings, key_list=key_list)
-        except exceptions.UnsupportedOperation:
-            if not everything:
-                log.warning("No portfolios in %s edition, audit of portfolios ignored", sq.edition())
-    if options.WHAT_APPS in what_to_audit:
-        try:
-            problems += applications.audit(endpoint=sq, audit_settings=settings, key_list=key_list)
-        except exceptions.UnsupportedOperation:
-            if not everything:
-                log.warning("No applications in %s edition, audit of portfolios ignored", sq.edition())
 
+    with util.open_file(settings.get("FILE", None), mode="w") as fd:
+        worker = Thread(target=write_problems, args=(q, fd, settings))
+        worker.daemon = True
+        worker.name = "WriteProblems"
+        worker.start()
+        if options.WHAT_PROJECTS in what_to_audit:
+            pbs = projects.audit(endpoint=sq, audit_settings=settings, key_list=key_list, write_q=q)
+            q.put(pbs)
+            problems += pbs
+        if options.WHAT_PROFILES in what_to_audit:
+            pbs = qualityprofiles.audit(endpoint=sq, audit_settings=settings)
+            q.put(pbs)
+            problems += pbs
+        if options.WHAT_GATES in what_to_audit:
+            pbs = qualitygates.audit(endpoint=sq, audit_settings=settings)
+            q.put(pbs)
+            problems += pbs
+        if options.WHAT_SETTINGS in what_to_audit:
+            pbs = sq.audit(audit_settings=settings)
+            q.put(pbs)
+            problems += pbs
+        if options.WHAT_USERS in what_to_audit:
+            pbs = users.audit(endpoint=sq, audit_settings=settings)
+            q.put(pbs)
+            problems += pbs
+        if options.WHAT_GROUPS in what_to_audit:
+            pbs = groups.audit(endpoint=sq, audit_settings=settings)
+            q.put(pbs)
+            problems += pbs
+        if options.WHAT_PORTFOLIOS in what_to_audit:
+            try:
+                pbs = portfolios.audit(endpoint=sq, audit_settings=settings, key_list=key_list)
+                q.put(pbs)
+                problems += pbs
+            except exceptions.UnsupportedOperation:
+                if not everything:
+                    log.warning("No portfolios in %s edition, audit of portfolios ignored", sq.edition())
+        if options.WHAT_APPS in what_to_audit:
+            try:
+                pbs = applications.audit(endpoint=sq, audit_settings=settings, key_list=key_list)
+                q.put(pbs)
+                problems += pbs
+            except exceptions.UnsupportedOperation:
+                if not everything:
+                    log.warning("No applications in %s edition, audit of portfolios ignored", sq.edition())
+        q.put(None)
+        q.join()
     return problems
 
 
@@ -138,16 +189,22 @@ def main():
         config.configure()
         sys.exit(0)
 
+    ofile = kwargs.pop(options.REPORT_FILE)
+    settings["FILE"] = ofile
+    settings["CSV_DELIMITER"] = kwargs[options.CSV_SEPARATOR]
+    settings["WITH_URL"] = kwargs[options.WITH_URL]
+
     if kwargs.get("sif", None) is not None:
         err = errcodes.SIF_AUDIT_ERROR
         try:
             (server_id, problems) = _audit_sif(kwargs["sif"], settings)
+            settings["SERVER_ID"] = server_id
         except json.decoder.JSONDecodeError:
             util.exit_fatal(f"File {kwargs['sif']} does not seem to be a legit JSON file, aborting...", err)
         except FileNotFoundError:
             util.exit_fatal(f"File {kwargs['sif']} does not exist, aborting...", err)
         except PermissionError:
-            util.exit_fatal(f"No permissiont to open file {kwargs['sif']}, aborting...", err)
+            util.exit_fatal(f"No permission to open file {kwargs['sif']}, aborting...", err)
         except sif.NotSystemInfo:
             util.exit_fatal(f"File {kwargs['sif']} does not seem to be a system info or support info file, aborting...", err)
     else:
@@ -157,6 +214,7 @@ def main():
         except exceptions.ConnectionError as e:
             util.exit_fatal(e.message, e.errcode)
         server_id = sq.server_id()
+        settings["SERVER_ID"] = server_id
         util.check_token(kwargs[options.TOKEN])
         key_list = kwargs[options.KEYS]
         if key_list is not None and len(key_list) > 0 and "projects" in util.csv_to_list(kwargs[options.WHAT]):
@@ -168,7 +226,6 @@ def main():
         except exceptions.ObjectNotFound as e:
             util.exit_fatal(e.message, errcodes.NO_SUCH_KEY)
 
-    ofile = kwargs.pop(options.REPORT_FILE)
     if problems:
         log.warning("%d issues found during audit", len(problems))
     else:
diff --git a/sonar/projects.py b/sonar/projects.py
@@ -643,7 +643,7 @@ def __audit_scanner(self, audit_settings: types.ConfigSettings) -> list[Problem]
             return []
         return [Problem(get_rule(RuleId.PROJ_WRONG_SCANNER), self, str(self), proj_type, scanner)]
 
-    def audit(self, audit_settings: types.ConfigSettings) -> list[Problem]:
+    def audit(self, audit_settings: types.ConfigSettings, write_q: Queue[list[Problem]]) -> list[Problem]:
         """Audits a project and returns the list of problems found
 
         :param dict audit_settings: Options of what to audit and thresholds to raise problems
@@ -666,6 +666,8 @@ def audit(self, audit_settings: types.ConfigSettings) -> list[Problem]:
             problems += self.__audit_scanner(audit_settings)
         except (ConnectionError, RequestException) as e:
             log.error("%s while auditing %s", util.error_msg(e), str(self))
+        if write_q:
+            write_q.put(problems)
         return problems
 
     def export_zip(self, timeout: int = 180) -> dict[str, str]:
@@ -1406,32 +1408,43 @@ def get_list(endpoint: pf.Platform, key_list: types.KeyList = None, use_cache: b
     return {key: Project.get_object(endpoint, key) for key in util.csv_to_list(key_list)}
 
 
-def __audit_thread(queue: Queue[Project], results: list[Problem], audit_settings: types.ConfigSettings, bindings: dict[str, str]) -> None:
+def __audit_thread(
+    queue: Queue[Project],
+    results: list[Problem],
+    audit_settings: types.ConfigSettings,
+    bindings: dict[str, str],
+    write_q: Optional[Queue[list[Problem]]],
+) -> None:
     """Audit callback function for multitheaded audit"""
     audit_bindings = audit_settings.get("audit.projects.bindings", True)
     while not queue.empty():
         log.debug("Picking from the queue")
         project = queue.get()
-        results += project.audit(audit_settings)
+        problems = project.audit(audit_settings, write_q)
         try:
             if project.endpoint.edition() == "community" or not audit_bindings or project.is_part_of_monorepo():
                 queue.task_done()
                 log.debug("%s audit done", str(project))
                 continue
             bindkey = project.binding_key()
             if bindkey and bindkey in bindings:
-                results.append(Problem(get_rule(RuleId.PROJ_DUPLICATE_BINDING), project, str(project), str(bindings[bindkey])))
+                problems.append(Problem(get_rule(RuleId.PROJ_DUPLICATE_BINDING), project, str(project), str(bindings[bindkey])))
             else:
                 bindings[bindkey] = project
         except (ConnectionError, RequestException) as e:
             log.error("%s while auditing %s", util.error_msg(e), str(project))
         __increment_processed(audit_settings)
+        if write_q:
+            write_q.put(problems)
+        results += problems
         queue.task_done()
 
     log.debug("Audit of projects completed")
 
 
-def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings, key_list: types.KeyList = None) -> list[Problem]:
+def audit(
+    endpoint: pf.Platform, audit_settings: types.ConfigSettings, key_list: types.KeyList = None, write_q: Optional[Queue[list[Problem]]] = None
+) -> list[Problem]:
     """Audits all or a list of projects
 
     :param Platform endpoint: reference to the SonarQube platform
@@ -1451,19 +1464,21 @@ def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings, key_list:
     bindings = {}
     for i in range(audit_settings.get("threads", 1)):
         log.debug("Starting project audit thread %d", i)
-        worker = Thread(target=__audit_thread, args=(q, problems, audit_settings, bindings))
+        worker = Thread(target=__audit_thread, args=(q, problems, audit_settings, bindings, write_q))
         worker.setDaemon(True)
         worker.setName(f"ProjectAudit{i}")
         worker.start()
     q.join()
     if not audit_settings.get("audit.projects.duplicates", True):
         log.info("Project duplicates auditing was disabled by configuration")
-        return problems
-    for key, p in plist.items():
-        log.debug("Auditing for potential duplicate projects")
-        for key2 in plist:
-            if key2 != key and re.match(key2, key):
-                problems.append(Problem(get_rule(RuleId.PROJ_DUPLICATE), p, str(p), key2))
+    else:
+        log.info("Auditing for potential duplicate projects")
+        for key, p in plist.items():
+            for key2 in plist:
+                if key2 != key and re.match(key2, key):
+                    problems.append(Problem(get_rule(RuleId.PROJ_DUPLICATE), p, str(p), key2))
+    if write_q:
+        write_q.put(problems)
     return problems
 
 
@@ -1508,7 +1523,7 @@ def export(
     q = Queue(maxsize=0)
     proj_list = get_list(endpoint=endpoint, key_list=key_list)
     export_settings["NBR_PROJECTS"] = len(proj_list)
-    export_settings["EXPORTED"] = 0
+    export_settings["PROCESSED"] = 0
     log.info("Exporting %d projects", export_settings["NBR_PROJECTS"])
     for p in proj_list.values():
         q.put(p)
