Solve-sonar-config-regressions (#1390)

* Fix queue export

* Remove useless import os

* Fix inlining

* Quality pass

* Allow output in stdout even in streaming mode

Author: okorach

cli/config.py

@@ -23,13 +23,15 @@
 """
 import sys
 import os
-from threading import Lock
+from threading import Thread, Lock
+from queue import Queue
 
 import json
 import yaml
 
 from cli import options
 from sonar import exceptions, errcodes, utilities
+from sonar.util import types
 import sonar.logging as log
 from sonar import platform, rules, qualityprofiles, qualitygates, users, groups
 from sonar import projects, portfolios, applications
@@ -219,44 +221,84 @@ def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs)
     log.info("Synchronous export of configuration from %s completed", kwargs["url"])
 
 
+def write_objects(queue: Queue, fd, object_type: str, export_settings: types.ConfigSettings) -> None:
+    """
+    Thread to write projects in the JSON file
+    """
+    done = False
+    prefix = ""
+    log.info("Waiting %s to write...", object_type)
+    print(f'"{object_type}": ' + "{", file=fd)
+    while not done:
+        obj_json = queue.get()
+        done = obj_json is None
+        if not done:
+            if export_settings.get("INLINE_LISTS", True):
+                obj_json = utilities.inline_lists(obj_json, exceptions=("conditions",))
+            if object_type in ("projects", "applications", "portfolios", "users"):
+                if object_type == "users":
+                    key = obj_json.pop("login", None)
+                else:
+                    key = obj_json.pop("key", None)
+                log.debug("Writing %s key '%s'", object_type[:-1], key)
+                print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
+            else:
+                log.debug("Writing %s", object_type)
+                print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
+            prefix = ",\n"
+        queue.task_done()
+    print("\n}", file=fd, end="")
+    log.info("Writing %s complete", object_type)
+
+
 def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
     """Exports a platform configuration in a JSON file"""
+    file = kwargs[options.REPORT_FILE]
     export_settings = {
         "INLINE_LISTS": not kwargs["dontInlineLists"],
-        "EXPORT_DEFAULTS": kwargs["exportDefaults"],
-        "FULL_EXPORT": kwargs["fullExport"],
+        "EXPORT_DEFAULTS": True,
+        # "FULL_EXPORT": kwargs["fullExport"],
+        "FULL_EXPORT": False,
+        "MODE": "MIGRATION",
         "THREADS": kwargs[options.NBR_THREADS],
-        options.REPORT_FILE: kwargs[options.REPORT_FILE],
-        "WRITE_CALLBACK": write_project,
+        "SKIP_ISSUES": True,
     }
-    log.info("Exporting configuration from %s (asynchronously)", kwargs[options.URL])
+    if "projects" in what and kwargs[options.KEYS]:
+        non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)]
+        if len(non_existing_projects) > 0:
+            utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY)
+
+    log.info("Exporting configuration from %s", kwargs[options.URL])
     key_list = kwargs[options.KEYS]
     sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()}
-    for what_item, call_data in _EXPORT_CALLS.items():
-        if what_item not in what or what_item == options.WHAT_PROJECTS:
-            continue
-        ndx, func = call_data
-        try:
-            sq_settings[ndx] = utilities.remove_empties(func(endpoint, export_settings=export_settings, key_list=key_list))
-            __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT])
-        except exceptions.UnsupportedOperation as e:
-            log.warning(e.message)
-        except exceptions.ObjectNotFound as e:
-            log.error(e.message)
-    if not kwargs["dontInlineLists"]:
-        sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",))
-    __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT])
-
-    __remove_chars_at_end(kwargs[options.REPORT_FILE], 3)
-    __add_project_header(kwargs[options.REPORT_FILE])
-    projects.export(endpoint, export_settings=export_settings, key_list=None)
-    __add_project_footer(kwargs[options.REPORT_FILE])
-    log.info("Asynchronous export of configuration from %s completed", kwargs["url"])
+    is_first = True
+    q = Queue(maxsize=0)
+    with utilities.open_file(file, mode="w") as fd:
+        print("{", file=fd)
+        for what_item, call_data in _EXPORT_CALLS.items():
+            if what_item not in what:
+                continue
+            ndx, func = call_data
+            try:
+                if not is_first:
+                    print(",", file=fd)
+                is_first = False
+                worker = Thread(target=write_objects, args=(q, fd, ndx, export_settings))
+                worker.daemon = True
+                worker.name = f"Write{ndx[:1].upper()}{ndx[1:10]}"
+                worker.start()
+                sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list, write_q=q)
+                q.join()
+            except exceptions.UnsupportedOperation as e:
+                log.warning(e.message)
+        sq_settings = utilities.remove_empties(sq_settings)
+        print("\n}", file=fd)
+    log.info("Exporting migration data from %s completed", kwargs["url"])
 
 
 def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
     """Exports the configuration of the SonarQube platform"""
-    if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what or kwargs[options.FORMAT] != "json" or not kwargs[options.REPORT_FILE]:
+    if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what or kwargs[options.FORMAT] != "json":
         __export_config_sync(endpoint=endpoint, what=what, **kwargs)
     else:
         __export_config_async(endpoint=endpoint, what=what, **kwargs)

migration/migration.py

@@ -22,7 +22,6 @@
     Exports SonarQube platform configuration as JSON
 """
 import sys
-import os
 from threading import Thread, Lock
 from queue import Queue