Fix sonar-config output like before (#1397)

* Remove unwanted properties in config mode

* Add JSON normalization

* Final file normalization

* Extract platform and set CONFIG mode

* Remove unused code

* Sort, remove empty and nones before dumping

* Remove premature removal of nones

* Fix QP diffing in inheritance

* Last fixes

* Reduce complexity

* Quality pass

* Quality pass

* Handle JSON decode error when normalizing JSON file

* Black'ing

Author: okorach

cli/config.py

@@ -22,7 +22,7 @@
     Exports SonarQube platform configuration as JSON
 """
 import sys
-import os
+from typing import TextIO
 from threading import Thread, Lock
 from queue import Queue
 
@@ -78,6 +78,7 @@
 
 
 _EXPORT_CALLS = {
+    "platform": [__JSON_KEY_PLATFORM, platform.basics],
     options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export],
     options.WHAT_RULES: [__JSON_KEY_RULES, rules.export],
     options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export],
@@ -135,36 +136,6 @@ def __write_export(config: dict[str, str], file: str, format: str) -> None:
             print(utilities.json_dump(config), file=fd)
 
 
-def __remove_chars_at_end(file: str, nb_bytes: int) -> None:
-    """Writes the configuration in file"""
-    with open(file, mode="rb+") as fd:
-        fd.seek(-nb_bytes, os.SEEK_END)
-        fd.truncate()
-
-
-def __add_project_header(file: str) -> None:
-    """Writes the configuration in file"""
-    with open(file, mode="a", encoding="utf-8") as fd:
-        print(',\n   "projects": {\n', file=fd)
-
-
-def __add_project_footer(file: str) -> None:
-    """Closes projects section"""
-    __remove_chars_at_end(file, 2)
-    with open(file, mode="a", encoding="utf-8") as fd:
-        print("\n   }\n}", file=fd)
-
-
-def write_project(project_json: dict[str, any], file: str) -> None:
-    """
-    writes a project JSON in a file
-    """
-    key = project_json.pop("key")
-    with _WRITE_LOCK:
-        with utilities.open_file(file, mode="a") as fd:
-            print(f'"{key}": {utilities.json_dump(project_json)},', file=fd)
-
-
 def __convert_for_yaml(json_export: dict[str, any]) -> dict[str, any]:
     """Converts the default JSON produced by export to a modified version more suitable for YAML"""
     if "globalSettings" in json_export:
@@ -203,7 +174,8 @@ def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs)
             utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY)
     log.info("Exporting configuration synchronously from %s", kwargs[options.URL])
     key_list = kwargs[options.KEYS]
-    sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()}
+    what.append("platform")
+    sq_settings = {}
     for what_item, call_data in _EXPORT_CALLS.items():
         if what_item not in what:
             continue
@@ -214,38 +186,45 @@ def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs)
             log.warning(e.message)
         except exceptions.ObjectNotFound as e:
             log.error(e.message)
-    sq_settings = utilities.remove_empties(sq_settings)
-    if not kwargs["dontInlineLists"]:
-        sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",))
+    sq_settings = __prep_json_for_write(sq_settings, export_settings)
     __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT])
     log.info("Synchronous export of configuration from %s completed", kwargs["url"])
 
 
-def write_objects(queue: Queue, fd, object_type: str, export_settings: types.ConfigSettings) -> None:
+def __prep_json_for_write(json_data: types.ObjectJsonRepr, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
+    """Cleans up the JSON before writing"""
+    json_data = utilities.sort_lists(json_data)
+    if not export_settings.get("FULL_EXPORT", False):
+        json_data = utilities.remove_empties(utilities.remove_nones(json_data))
+    if export_settings.get("INLINE_LISTS", True):
+        json_data = utilities.inline_lists(json_data, exceptions=("conditions",))
+    return json_data
+
+
+def write_objects(queue: Queue[types.ObjectJsonRepr], fd: TextIO, object_type: str, export_settings: types.ConfigSettings) -> None:
     """
     Thread to write projects in the JSON file
     """
-    done = False
     prefix = ""
     log.info("Waiting %s to write...", object_type)
     print(f'"{object_type}": ' + "{", file=fd)
-    while not done:
+    while True:
         obj_json = queue.get()
-        done = obj_json is None
-        if not done:
-            if export_settings.get("INLINE_LISTS", True):
-                obj_json = utilities.inline_lists(obj_json, exceptions=("conditions",))
-            if object_type in ("projects", "applications", "portfolios", "users"):
-                if object_type == "users":
-                    key = obj_json.pop("login", None)
-                else:
-                    key = obj_json.pop("key", None)
-                log.debug("Writing %s key '%s'", object_type[:-1], key)
-                print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
+        if obj_json is None:
+            queue.task_done()
+            break
+        obj_json = __prep_json_for_write(obj_json, export_settings)
+        if object_type in ("projects", "applications", "portfolios", "users"):
+            if object_type == "users":
+                key = obj_json.pop("login", None)
             else:
-                log.debug("Writing %s", object_type)
-                print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
-            prefix = ",\n"
+                key = obj_json.pop("key", None)
+            log.debug("Writing %s key '%s'", object_type[:-1], key)
+            print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
+        else:
+            log.debug("Writing %s", object_type)
+            print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
+        prefix = ",\n"
         queue.task_done()
     print("\n}", file=fd, end="")
     log.info("Writing %s complete", object_type)
@@ -256,10 +235,9 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs
     file = kwargs[options.REPORT_FILE]
     export_settings = {
         "INLINE_LISTS": not kwargs["dontInlineLists"],
-        "EXPORT_DEFAULTS": True,
-        # "FULL_EXPORT": kwargs["fullExport"],
-        "FULL_EXPORT": False,
-        "MODE": "MIGRATION",
+        "EXPORT_DEFAULTS": kwargs["exportDefaults"],
+        "FULL_EXPORT": kwargs["fullExport"],
+        "MODE": "CONFIG",
         "THREADS": kwargs[options.NBR_THREADS],
         "SKIP_ISSUES": True,
     }
@@ -270,7 +248,7 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs
 
     log.info("Exporting configuration from %s", kwargs[options.URL])
     key_list = kwargs[options.KEYS]
-    sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()}
+    what.append("platform")
     is_first = True
     q = Queue(maxsize=0)
     with utilities.open_file(file, mode="w") as fd:
@@ -287,12 +265,12 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs
                 worker.daemon = True
                 worker.name = f"Write{ndx[:1].upper()}{ndx[1:10]}"
                 worker.start()
-                sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list, write_q=q)
+                func(endpoint, export_settings=export_settings, key_list=key_list, write_q=q)
                 q.join()
             except exceptions.UnsupportedOperation as e:
                 log.warning(e.message)
-        sq_settings = utilities.remove_empties(sq_settings)
         print("\n}", file=fd)
+    utilities.normalize_json_file(file, remove_empty=True, remove_none=True)
     log.info("Exporting migration data from %s completed", kwargs["url"])
 
 

migration/migration.py

@@ -181,6 +181,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
         # if not kwargs.get("dontInlineLists", False):
         #    sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",))
         print("\n}", file=fd)
+    utilities.normalize_json_file(file, remove_empty=False, remove_none=True)
     log.info("Exporting migration data from %s completed", kwargs["url"])
 
 

sonar/qualityprofiles.py

@@ -381,7 +381,7 @@ def _treat_modified_rules(self, modified_rules: dict[str:str]) -> dict[str:str]:
                     diff_rules[r_key]["params"].update(parms)
         return diff_rules
 
-    def diff(self, another_qp: QualityProfile, qp_json_data: dict[str:str] = None) -> tuple[dict[str:str], dict[str:str]]:
+    def diff(self, another_qp: QualityProfile, qp_json_data: dict[str:str]) -> tuple[dict[str:str], dict[str:str]]:
         """Returns the list of rules added or modified in self compared to another_qp (for inheritance)
         :param another_qp: The second quality profile to diff
         :type another_qp: QualityProfile
@@ -401,8 +401,6 @@ def diff(self, another_qp: QualityProfile, qp_json_data: dict[str:str] = None) -
             diff_rules["removedRules"] = {}
 
         log.debug("Returning QP diff %s", str(diff_rules))
-        if qp_json_data is None:
-            return (diff_rules, qp_json_data)
         for index in ("addedRules", "modifiedRules", "removedRules"):
             if index not in diff_rules:
                 continue
@@ -549,7 +547,7 @@ def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings = None) ->
     return problems
 
 
-def hierarchize_language(qp_list: dict[str, str]) -> types.ObjectJsonRepr:
+def hierarchize_language(qp_list: dict[str, str], endpoint: pf.Platform, language: str) -> types.ObjectJsonRepr:
     """Organizes a flat list of quality profiles in inheritance hierarchy"""
     log.debug("Organizing QP list %s in hierarchy", str(qp_list.keys()))
     hierarchy = qp_list.copy()
@@ -559,17 +557,21 @@ def hierarchize_language(qp_list: dict[str, str]) -> types.ObjectJsonRepr:
             if qp_json_data["parentName"] not in hierarchy:
                 log.critical("Can't find parent %s in quality profiles", qp_json_data["parentName"])
                 continue
-            parent_qp = hierarchy[qp_json_data.pop("parentName")]
+            parent_qp_name = qp_json_data.pop("parentName")
+            parent_qp = hierarchy[parent_qp_name]
             if _CHILDREN_KEY not in parent_qp:
                 parent_qp[_CHILDREN_KEY] = {}
-            parent_qp[_CHILDREN_KEY][qp_name] = qp_json_data
+            this_qp = get_object(endpoint=endpoint, name=qp_name, language=language)
+            (_, diff_data) = this_qp.diff(get_object(endpoint=endpoint, name=parent_qp_name, language=language), qp_json_data)
+            diff_data.pop("rules", None)
+            parent_qp[_CHILDREN_KEY][qp_name] = diff_data
             to_remove.append(qp_name)
     for qp_name in to_remove:
         hierarchy.pop(qp_name)
     return hierarchy
 
 
-def hierarchize(qp_list: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+def hierarchize(qp_list: types.ObjectJsonRepr, endpoint: pf.Platform) -> types.ObjectJsonRepr:
     """Organize a flat list of QP in hierarchical (inheritance) fashion
 
     :param qp_list: List of quality profiles
@@ -580,7 +582,7 @@ def hierarchize(qp_list: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
     log.info("Organizing quality profiles in hierarchy")
     hierarchy = {}
     for lang, lang_qp_list in qp_list.items():
-        hierarchy[lang] = hierarchize_language(lang_qp_list)
+        hierarchy[lang] = hierarchize_language(lang_qp_list, endpoint=endpoint, language=lang)
     return hierarchy
 
 
@@ -604,7 +606,7 @@ def export(
         if lang not in qp_list:
             qp_list[lang] = {}
         qp_list[lang][name] = json_data
-    qp_list = hierarchize(qp_list)
+    qp_list = hierarchize(qp_list, endpoint=endpoint)
     if write_q:
         write_q.put(qp_list)
         write_q.put(None)

sonar/users.py

@@ -389,7 +389,9 @@ def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr
 
         if not self.endpoint.is_sonarcloud() and not export_settings["FULL_EXPORT"] and not json_data["local"]:
             json_data.pop("local")
-        return util.remove_nones(util.filter_export(json_data, SETTABLE_PROPERTIES, export_settings["FULL_EXPORT"]))
+        for key in "sonarQubeLastConnectionDate", "externalLogin", "externalProvider", "id", "managed":
+            json_data.pop(key, None)
+        return util.filter_export(json_data, SETTABLE_PROPERTIES, export_settings["FULL_EXPORT"])
 
 
 def search(endpoint: pf.Platform, params: types.ApiParams = None) -> dict[str, User]:

sonar/utilities.py

@@ -185,6 +185,18 @@ def remove_empties(d: dict[str, any]) -> dict[str, any]:
     return new_d
 
 
+def sort_lists(d: dict[str, any]) -> dict[str, any]:
+    """Recursively removes empty lists and dicts and none from a dict"""
+    # log.debug("Cleaning up %s", json_dump(d))
+    new_d = d.copy()
+    for k, v in d.items():
+        if isinstance(v, list) and len(v) > 0 and isinstance(v[0], (str, int, float)):
+            new_d[k] = sorted(v)
+        elif isinstance(v, dict):
+            new_d[k] = sort_lists(v)
+    return new_d
+
+
 def dict_subset(d: dict[str, str], subset_list: list[str]) -> dict[str, str]:
     """Returns the subset of dict only with subset_list keys"""
     return {key: d[key] for key in subset_list if key in d}
@@ -197,7 +209,7 @@ def allowed_values_string(original_str: str, allowed_values: list[str]) -> str:
 
 def json_dump(jsondata: Union[list[str], dict[str, str]], indent: int = 3) -> str:
     """JSON dump helper"""
-    return json.dumps(remove_nones(jsondata), indent=indent, sort_keys=True, separators=(",", ": "))
+    return json.dumps(jsondata, indent=indent, sort_keys=True, separators=(",", ": "))
 
 
 def csv_to_list(string: str, separator: str = ",") -> list[str]:
@@ -651,3 +663,23 @@ def list_to_dict(original_list: list[dict[str, any]], key_field: str) -> dict[st
 def dict_to_list(original_dict: dict[str, any], key_field: str, value_field: Optional[str] = "value") -> list[str, any]:
     """Converts a dict to list adding dict key in list key_field"""
     return [{key_field: key, value_field: elem} if not isinstance(elem, dict) else {key_field: key, **elem} for key, elem in original_dict.items()]
+
+
+def normalize_json_file(file: Optional[str], remove_empty: bool = True, remove_none: bool = True) -> None:
+    """Sorts a JSON file and optionally remove empty and none values"""
+    if file is None:
+        log.info("Output is stdout, skipping normalization")
+        return
+    log.info("Normalizing JSON file '%s'", file)
+    try:
+        with open_file(file, mode="r") as fd:
+            json_data = json.loads(fd.read())
+    except json.decoder.JSONDecodeError:
+        log.warning("JSON Decode error while normalizing file '%s', is file complete?", file)
+        return
+    if remove_empty:
+        json_data = remove_empties(json_data)
+    if remove_none:
+        json_data = remove_nones(json_data)
+    with open_file(file, mode="w") as fd:
+        print(json_dump(json_data), file=fd)