Revert "Change-sonar-config-format (#2040)"

This reverts commit 4651e56.

Author: okorach-sonar

cli/config.py

@@ -22,7 +22,7 @@
 Exports SonarQube platform configuration as JSON
 """
 
-from typing import TextIO, Any
+from typing import TextIO
 from threading import Thread
 from queue import Queue
 
@@ -122,20 +122,16 @@ def __parse_args(desc: str) -> object:
     return options.parse_and_check(parser=parser, logger_name=TOOL_NAME)
 
 
-def __normalize_json(json_data: dict[str, Any], remove_empty: bool = True, remove_none: bool = True) -> dict[str, any]:
+def __normalize_json(json_data: dict[str, any], remove_empty: bool = True, remove_none: bool = True) -> dict[str, any]:
     """Sorts a JSON file and optionally remove empty and none values"""
     log.info("Normalizing JSON - remove empty = %s, remove nones = %s", str(remove_empty), str(remove_none))
-    json_data = utilities.clean_data(json_data, remove_empty, remove_none)
+    if remove_empty:
+        json_data = utilities.remove_empties(json_data)
+    if remove_none:
+        json_data = utilities.remove_nones(json_data)
     json_data = utilities.order_keys(json_data, *_SECTIONS_ORDER)
     for key in [k for k in _SECTIONS_TO_SORT if k in json_data]:
-        if isinstance(json_data[key], (list, tuple, set)):
-            if len(json_data[key]) > 0:
-                sort_field = next((k for k in ("key", "name", "login") if k in json_data[key][0]), None)
-                if sort_field:
-                    tmp_d = {v[sort_field]: v for v in json_data[key]}
-                    json_data[key] = list(dict(sorted(tmp_d.items())).values())
-        else:
-            json_data[key] = {k: json_data[key][k] for k in sorted(json_data[key])}
+        json_data[key] = {k: json_data[key][k] for k in sorted(json_data[key])}
     return json_data
 
 
@@ -175,37 +171,28 @@ def write_objects(queue: Queue[types.ObjectJsonRepr], fd: TextIO, object_type: s
     """
     done = False
     prefix = ""
-    objects_exported_as_lists = ("projects", "applications", "users", "portfolios")
-    objects_exported_as_whole = ("qualityGates", "groups")
     log.info("Waiting %s to write...", object_type)
-    if object_type in objects_exported_as_lists:
-        start, stop = ("[", "]")
-    elif object_type in objects_exported_as_whole:
-        start, stop = ("", "")
-    else:
-        start, stop = ("{", "}")
-    print(f'"{object_type}": ' + start, file=fd)
+    print(f'"{object_type}": ' + "{", file=fd)
     while not done:
         obj_json = queue.get()
         if not (done := obj_json is utilities.WRITE_END):
             if object_type == "groups":
                 obj_json = __prep_json_for_write(obj_json, {**export_settings, EXPORT_EMPTY: True})
             else:
                 obj_json = __prep_json_for_write(obj_json, export_settings)
-            key = "" if isinstance(obj_json, list) else obj_json.get("key", obj_json.get("login", obj_json.get("name", "unknown")))
-            log.debug("Writing %s key '%s'", object_type[:-1], key)
-            if object_type in objects_exported_as_lists:
-                print(f"{prefix}{utilities.json_dump(obj_json)}", end="", file=fd)
-            elif object_type in objects_exported_as_whole:
-                print(f"{prefix}{utilities.json_dump(obj_json)}", end="", file=fd)
-            elif object_type in ("applications", "portfolios", "users"):
+            if object_type in ("projects", "applications", "portfolios", "users"):
+                if object_type == "users":
+                    key = obj_json.pop("login", None)
+                else:
+                    key = obj_json.pop("key", None)
+                log.debug("Writing %s key '%s'", object_type[:-1], key)
                 print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
             else:
                 log.debug("Writing %s", object_type)
                 print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
             prefix = ",\n"
         queue.task_done()
-    print("\n" + stop, file=fd, end="")
+    print("\n}", file=fd, end="")
     log.info("Writing %s complete", object_type)
 
 
@@ -270,7 +257,10 @@ def __prep_json_for_write(json_data: types.ObjectJsonRepr, export_settings: type
     if export_settings.get("MODE", "CONFIG") == "MIGRATION":
         return json_data
     if not export_settings.get("FULL_EXPORT", False):
-        json_data = utilities.clean_data(json_data, remove_empty=not export_settings.get(EXPORT_EMPTY, False), remove_none=True)
+        json_data = utilities.remove_nones(json_data)
+        if not export_settings.get(EXPORT_EMPTY, False):
+            log.debug("Removing empties")
+            json_data = utilities.remove_empties(json_data)
     if export_settings.get("INLINE_LISTS", True):
         json_data = utilities.inline_lists(json_data, exceptions=("conditions",))
     return json_data

conf/build.sh

@@ -21,7 +21,6 @@ CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 build_docs=0
 build_docker=0
-offline=0
 
 . "${CONF_DIR}/env.sh"
 
@@ -33,9 +32,6 @@ while [[ $# -ne 0 ]]; do
         docker)
             build_docker=1
             ;;
-        offline)
-            offline=1
-            ;;
         *)
             ;;
     esac
@@ -45,18 +41,9 @@ done
 echo "======= FORMATTING CODE ========="
 ruff format
 echo "======= BUILDING PACKAGE ========="
-if [[ "${offline}" = "1" ]]; then
-    cp "${ROOT_DIR}/conf/offline/setup.py" "${ROOT_DIR}/"
-    cp "${ROOT_DIR}/conf/offline/sonar-tools" "${ROOT_DIR}/"
-    mv "${ROOT_DIR}/pyproject.toml" "${ROOT_DIR}/pyproject.toml.bak"
-    python setup.py bdist_wheel
-    mv "${ROOT_DIR}/pyproject.toml.bak" "${ROOT_DIR}/pyproject.toml"
-    rm "${ROOT_DIR}/setup.py" "${ROOT_DIR}/sonar-tools"
-    # python -m build
-else
-    rm -rf "${ROOT_DIR}/build/lib/sonar" "${ROOT_DIR}/build/lib/cli" "${ROOT_DIR}"/build/scripts*/sonar-tools "${ROOT_DIR}"/dist/sonar_tools*
-    poetry build
-fi
+rm -rf "${ROOT_DIR}/build/lib/sonar" "${ROOT_DIR}/build/lib/cli" "${ROOT_DIR}"/build/scripts*/sonar-tools "${ROOT_DIR}"/dist/sonar_tools*
+# python -m build
+poetry build
 
 if [[ "${build_docs}" = "1" ]]; then
     echo "======= BUILDING DOCS ========="

conf/offline/setup.py

@@ -171,13 +171,10 @@ def export(self) -> types.ObjectJsonRepr:
         :param full: Whether to do a full export including settings that can't be set, defaults to False
         :type full: bool, optional
         """
-        log.info("Exporting %s from %s", self, utilities.json_dump(self.sq_json))
-        jsondata = {"name": self.name}
+        log.info("Exporting %s from %s", self, self.sq_json)
+        jsondata = {"projects": {b["key"]: b["branch"] if b["selected"] else utilities.DEFAULT for b in self.sq_json["projects"]}}
         if self.is_main():
             jsondata["isMain"] = True
-        br_projects = [b for b in self.sq_json["projects"] if b.get("selected", True)]
-        br_projects = [{"key": b["key"], "branch": None if b["isMain"] else b["branch"]} for b in br_projects]
-        jsondata["projects"] = utilities.remove_nones(br_projects)
         return jsondata
 
     def update(self, name: str, project_branches: list[Branch]) -> bool:

conf/offline/sonar-tools

@@ -339,13 +339,12 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
                 "description": None if self._description == "" else self._description,
                 "visibility": self.visibility(),
                 # 'projects': self.projects(),
-                "branches": [br.export() for br in self.branches().values()],
+                "branches": {br.name: br.export() for br in self.branches().values()},
                 "permissions": self.permissions().export(export_settings=export_settings),
                 "tags": self.get_tags(),
             }
         )
-        json_data = util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))
-        return util.clean_data(json_data)
+        return util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))
 
     def set_permissions(self, data: types.JsonPermissions) -> application_permissions.ApplicationPermissions:
         """Sets an application permissions
@@ -526,9 +525,11 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, **kwarg
         app_json = app.export(export_settings)
         if write_q:
             write_q.put(app_json)
-        apps_settings[k] = app_json
+        else:
+            app_json.pop("key")
+            apps_settings[k] = app_json
     write_q and write_q.put(util.WRITE_END)
-    return dict(sorted(app_json.items())).values()
+    return apps_settings
 
 
 def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings, **kwargs) -> list[problem.Problem]:
@@ -601,4 +602,12 @@ def search_by_name(endpoint: pf.Platform, name: str) -> dict[str, Application]:
 
 def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
     """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
-    return original_json
+    new_json = util.dict_to_list(util.remove_nones(original_json), "key")
+    for app_json in new_json:
+        app_json["branches"] = util.dict_to_list(app_json["branches"], "name")
+        for b in app_json["branches"]:
+            if "projects" in b:
+                b["projects"] = [{"key": k, "branch": br} for k, br in b["projects"].items()]
+        if "permissions" in app_json:
+            app_json["permissions"] = permissions.convert_for_yaml(app_json["permissions"])
+    return new_json

sonar/app_branches.py

@@ -239,8 +239,12 @@ def export(endpoint: platform.Platform, export_settings: types.ConfigSettings) -
     :meta private:
     """
     log.info("Exporting DevOps integration settings")
-    devops_list = {s.key: s.to_json(export_settings) for s in get_list(endpoint).values()}
-    return list(dict(sorted(devops_list.items())).values())
+    json_data = {}
+    for s in get_list(endpoint).values():
+        export_data = s.to_json(export_settings)
+        key = export_data.pop("key")
+        json_data[key] = export_data
+    return json_data
 
 
 def import_config(endpoint: platform.Platform, config_data: types.ObjectJsonRepr, key_list: types.KeyList = None) -> int:

sonar/applications.py

@@ -345,12 +345,11 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, **kwarg
     """
 
     log.info("Exporting groups")
-    g_list = []
+    g_list = {}
     for g_name, g_obj in get_list(endpoint=endpoint).items():
         if not export_settings.get("FULL_EXPORT", False) and g_obj.is_default():
             continue
-        g_list.append({"name": g_name, "description": g_obj.description})
-    util.clean_data(g_list, remove_empty=False)
+        g_list[g_name] = "" if g_obj.description is None else g_obj.description
     log.info("%s groups to export", len(g_list))
     if write_q := kwargs.get("write_q", None):
         write_q.put(g_list)