Improve-json-format-conversion-code (#2072)

* Move stuff in platform_helper

* Use helpers and change conversion function names

* Change conversion function name

* Use the project conversion function

* Output report in deterministic order

* Add portfolio JSON format conversion

* AICode fix is a setting like others

* Handle "severity" like "severities"

* Add conversion of settings string value into float or int or bool

* Remove groups or users with no permissions in export

* Remove list inlining

* Remove perms field for objects with no permission

* Fix export for SQC

* Quality pass

Author: okorach

cli/config.py

@@ -32,14 +32,19 @@
 from cli import options
 from sonar import exceptions, errcodes, utilities, version
 from sonar.util import types, constants as c
+from sonar.util import platform_helper as pfhelp
+from sonar.util import project_helper as pjhelp
+from sonar.util import portfolio_helper as foliohelp
+from sonar.util import qualityprofile_helper as qphelp
+from sonar.util import rule_helper as rhelp
+
 import sonar.logging as log
 from sonar import platform, rules, qualityprofiles, qualitygates, users, groups
 from sonar import projects, portfolios, applications
 from sonar.util import component_helper
 
 TOOL_NAME = "sonar-config"
 
-DONT_INLINE_LISTS = "dontInlineLists"
 FULL_EXPORT = "fullExport"
 EXPORT_EMPTY = "exportEmpty"
 
@@ -104,14 +109,6 @@ def __parse_args(desc: str) -> object:
         f"By default the export will show the value as '{utilities.DEFAULT}' "
         "and the setting will not be imported at import time",
     )
-    parser.add_argument(
-        f"--{DONT_INLINE_LISTS}",
-        required=False,
-        default=False,
-        action="store_true",
-        help="By default, sonar-config exports multi-valued settings as comma separated strings instead of arrays (if there is not comma in values). "
-        "Set this flag if you want to force export multi valued settings as arrays",
-    )
     parser.add_argument(
         f"--{EXPORT_EMPTY}",
         required=False,
@@ -210,7 +207,6 @@ def export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> Non
     export_settings = kwargs.copy()
     export_settings.update(
         {
-            "INLINE_LISTS": not kwargs.get(DONT_INLINE_LISTS, False),
             "EXPORT_DEFAULTS": True,
             "FULL_EXPORT": kwargs.get(FULL_EXPORT, False),
             "MODE": mode,
@@ -268,8 +264,6 @@ def __prep_json_for_write(json_data: types.ObjectJsonRepr, export_settings: type
         if not export_settings.get(EXPORT_EMPTY, False):
             log.debug("Removing empties")
             json_data = utilities.clean_data(json_data, remove_empty=True)
-    if export_settings.get("INLINE_LISTS", True):
-        json_data = utilities.inline_lists(json_data, exceptions=("conditions",))
     return json_data
 
 
@@ -309,22 +303,23 @@ def convert_json(**kwargs) -> dict[str, Any]:
     with open(kwargs["convertFrom"], encoding="utf-8") as fd:
         old_json = json.loads(fd.read())
     mapping = {
-        "platform": platform.old_to_new_json,
-        "globalSettings": platform.global_settings_old_to_new_json,
-        "qualityProfiles": qualityprofiles.old_to_new_json,
-        "qualityGates": qualitygates.old_to_new_json,
-        "projects": projects.old_to_new_json,
-        "portfolios": portfolios.old_to_new_json,
-        "applications": applications.old_to_new_json,
-        "users": users.old_to_new_json,
-        "groups": groups.old_to_new_json,
-        "rules": rules.old_to_new_json,
+        "platform": pfhelp.convert_basics_json,
+        "globalSettings": pfhelp.convert_global_settings_json,
+        "qualityGates": qualitygates.convert_qgs_json,
+        "qualityProfiles": qphelp.convert_qps_json,
+        "projects": pjhelp.convert_projects_json,
+        "portfolios": foliohelp.convert_portfolios_json,
+        "applications": applications.convert_apps_json,
+        "users": users.convert_users_json,
+        "groups": groups.convert_groups_json,
+        "rules": rhelp.convert_rules_json,
     }
     new_json = {}
     for k, func in mapping.items():
         if k in old_json:
             log.info("Converting %s", k)
             new_json[k] = func(old_json[k])
+    new_json = __normalize_json(new_json, remove_empty=False, remove_none=True)
     with open(kwargs["convertTo"], mode="w", encoding="utf-8") as fd:
         print(utilities.json_dump(new_json), file=fd)
     return new_json

sonar/applications.py

@@ -42,6 +42,7 @@
 import sonar.utilities as util
 from sonar.audit import rules, problem
 import sonar.util.constants as c
+from sonar.util import common_json_helper
 
 _CLASS_LOCK = Lock()
 _IMPORTABLE_PROPERTIES = ("key", "name", "description", "visibility", "branches", "permissions", "tags")
@@ -344,7 +345,7 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
                 "tags": self.get_tags(),
             }
         )
-        json_data = old_to_new_json_one(json_data)
+        json_data = convert_app_json(json_data)
         return util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))
 
     def set_permissions(self, data: types.JsonPermissions) -> application_permissions.ApplicationPermissions:
@@ -518,7 +519,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, **kwarg
     """
     check_supported(endpoint)
     write_q = kwargs.get("write_q", None)
-    key_regexp = kwargs.get("key_list", ".*")
+    key_regexp = kwargs.get("key_list", ".+")
 
     app_list = {k: v for k, v in get_list(endpoint).items() if not key_regexp or re.match(key_regexp, k)}
     apps_settings = []
@@ -547,7 +548,7 @@ def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings, **kwargs)
         return []
     log.info("--- Auditing applications ---")
     problems = []
-    key_regexp = kwargs.get("key_list", None) or ".*"
+    key_regexp = kwargs.get("key_list", ".+")
     for obj in [o for o in get_list(endpoint).values() if not key_regexp or re.match(key_regexp, o.key)]:
         problems += obj.audit(audit_settings, **kwargs)
     return problems
@@ -600,19 +601,17 @@ def search_by_name(endpoint: pf.Platform, name: str) -> dict[str, Application]:
     return data
 
 
-def old_to_new_json_one(old_app_json: dict[str, Any]) -> dict[str, Any]:
+def convert_app_json(old_app_json: dict[str, Any]) -> dict[str, Any]:
     """Converts sonar-config old JSON report format to new format for a single application"""
-    new_json = old_app_json.copy()
-    if "permissions" in old_app_json:
-        new_json["permissions"] = util.perms_to_list(old_app_json["permissions"])
+    new_json = common_json_helper.convert_common_fields(old_app_json.copy())
     if "branches" in old_app_json:
         new_json["branches"] = util.dict_to_list(old_app_json["branches"], "name")
     return new_json
 
 
-def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]:
+def convert_apps_json(old_json: dict[str, Any]) -> dict[str, Any]:
     """Converts sonar-config old JSON report format to new format"""
     new_json = old_json.copy()
     for k, v in new_json.items():
-        new_json[k] = old_to_new_json_one(v)
+        new_json[k] = convert_app_json(v)
     return util.dict_to_list(new_json, "key")

sonar/groups.py

@@ -434,6 +434,6 @@ def exists(endpoint: pf.Platform, name: str) -> bool:
     return Group.get_object(endpoint=endpoint, name=name) is not None
 
 
-def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]:
+def convert_groups_json(old_json: dict[str, Any]) -> dict[str, Any]:
     """Converts sonar-config old groups JSON report format to new format"""
     return util.dict_to_list(old_json, "name", "description")

sonar/permissions/permission_templates.py

@@ -21,13 +21,14 @@
 """Abstraction of the SonarQube permission template concept"""
 
 from __future__ import annotations
-from typing import Optional
+from typing import Optional, Any
 
 import json
 import re
 
 import sonar.logging as log
 from sonar.util import types, cache
+from sonar.util import platform_helper as phelp
 from sonar import sqobject, utilities, exceptions
 from sonar.permissions import template_permissions
 import sonar.platform as pf
@@ -42,8 +43,6 @@
 _CREATE_API = "permissions/create_template"
 _UPDATE_API = "permissions/update_template"
 
-_IMPORTABLE_PROPERTIES = ("name", "description", "pattern", "defaultFor", "permissions")
-
 
 class PermissionTemplate(sqobject.SqObject):
     """Abstraction of the Sonar permission template concept"""
@@ -183,7 +182,7 @@ def to_json(self, export_settings: types.ConfigSettings = None) -> types.ObjectJ
             json_data.pop("pattern")
         json_data["creationDate"] = utilities.date_to_string(self.creation_date)
         json_data["lastUpdate"] = utilities.date_to_string(self.last_update)
-        return utilities.remove_nones(utilities.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False)))
+        return phelp.convert_template_json(json_data, export_settings.get("FULL_EXPORT", False))
 
     def _audit_pattern(self, audit_settings: types.ConfigSettings) -> list[pb.Problem]:
         log.debug("Auditing %s projectKeyPattern ('%s')", str(self), str(self.project_key_pattern))

sonar/permissions/permissions.py

@@ -86,10 +86,14 @@ def to_json(self, perm_type: str | None = None, csv: bool = False) -> types.Json
         if not csv:
             return self.permissions.get(perm_type, {}) if is_valid(perm_type) else self.permissions
         perms = {}
-        for p in normalize(perm_type):
-            if p not in self.permissions or len(self.permissions[p]) == 0:
+        for ptype in normalize(perm_type):
+            for k, v in self.permissions.get(ptype, {}).copy().items():
+                if len(v) == 0:
+                    self.permissions[ptype].pop(k)
+        for ptype in normalize(perm_type):
+            if ptype not in self.permissions or len(self.permissions[ptype]) == 0:
                 continue
-            perms[p] = {k: encode(v) for k, v in self.permissions.get(p, {}).items()}
+            perms[ptype] = {k: encode(v) for k, v in self.permissions.get(ptype, {}).items()}
         return perms if len(perms) > 0 else None
 
     def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:

sonar/platform.py

@@ -40,6 +40,7 @@
 import sonar.utilities as util
 from sonar.util import types, update_center
 import sonar.util.constants as c
+import sonar.util.platform_helper as pfhelp
 
 from sonar import errcodes, settings, devops, version, sif, exceptions, organizations
 from sonar.permissions import permissions, global_permissions, permission_templates
@@ -262,7 +263,7 @@ def delete(self, api: str, params: types.ApiParams = None, **kwargs) -> requests
     def __run_request(self, request: callable, api: str, params: types.ApiParams = None, **kwargs) -> requests.Response:
         """Makes an HTTP request to SonarQube"""
         mute = kwargs.pop("mute", ())
-        api = _normalize_api(api)
+        api = pfhelp.normalize_api(api)
         headers = {"user-agent": self._user_agent, "accept": _APP_JSON} | kwargs.get("headers", {})
         params = params or {}
         with_org = kwargs.pop("with_organization", True)
@@ -498,21 +499,7 @@ def export(self, export_settings: types.ConfigSettings, full: bool = False) -> t
         if not self.is_sonarcloud():
             json_data[settings.DEVOPS_INTEGRATION] = devops.export(self, export_settings=export_settings)
 
-        # Convert dicts to lists
-        special_categories = (settings.LANGUAGES_SETTINGS, settings.DEVOPS_INTEGRATION, "permissions", "permissionTemplates")
-        for categ in [cat for cat in settings.CATEGORIES if cat not in special_categories]:
-            json_data[categ] = util.sort_list_by_key(util.dict_to_list(json_data[categ], "key"), "key")
-        for k, v in json_data[settings.LANGUAGES_SETTINGS].items():
-            json_data[settings.LANGUAGES_SETTINGS][k] = util.sort_list_by_key(util.dict_to_list(v, "key"), "key")
-        json_data[settings.LANGUAGES_SETTINGS] = util.dict_to_list(json_data[settings.LANGUAGES_SETTINGS], "language", "settings")
-        json_data[settings.DEVOPS_INTEGRATION] = util.dict_to_list(json_data[settings.DEVOPS_INTEGRATION], "key")
-        json_data["permissions"] = util.perms_to_list(json_data["permissions"])
-        for v in json_data["permissionTemplates"].values():
-            if "permissions" in v:
-                v["permissions"] = util.perms_to_list(v["permissions"])
-        json_data["permissionTemplates"] = util.dict_to_list(json_data["permissionTemplates"], "key")
-
-        return util.order_dict(json_data, [*settings.CATEGORIES, "permissions", "permissionTemplates"])
+        return pfhelp.convert_global_settings_json(json_data)
 
     def set_webhooks(self, webhooks_data: types.ObjectJsonRepr) -> bool:
         """Sets global webhooks with a list of webhooks represented as JSON
@@ -791,19 +778,6 @@ def set_standard_experience(self) -> bool:
 this.context = Platform(os.getenv("SONAR_HOST_URL", "http://localhost:9000"), os.getenv("SONAR_TOKEN", ""))
 
 
-def _normalize_api(api: str) -> str:
-    """Normalizes an API based on its multiple original forms"""
-    if api.startswith("/api/"):
-        pass
-    elif api.startswith("api/"):
-        api = "/" + api
-    elif api.startswith("/"):
-        api = "/api" + api
-    else:
-        api = "/api/" + api
-    return api
-
-
 def _audit_setting_value(key: str, platform_settings: dict[str, Any], audit_settings: types.ConfigSettings, url: str) -> list[Problem]:
     """Audits a particular platform setting is set to expected value"""
     if (v := _get_multiple_values(4, audit_settings[key], "MEDIUM", "CONFIGURATION")) is None:
@@ -963,29 +937,3 @@ def audit(endpoint: Platform, audit_settings: types.ConfigSettings, **kwargs) ->
     pbs = endpoint.audit(audit_settings)
     "write_q" in kwargs and kwargs["write_q"].put(pbs)
     return pbs
-
-
-def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]:
-    """Converts sonar-config "plaform" section old JSON report format to new format"""
-    if "plugins" in old_json:
-        old_json["plugins"] = util.dict_to_list(old_json["plugins"], "key")
-    return old_json
-
-
-def global_settings_old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]:
-    """Converts sonar-config "globalSettings" section old JSON report format to new format"""
-    new_json = {}
-    special_categories = (settings.LANGUAGES_SETTINGS, settings.DEVOPS_INTEGRATION, "permissions", "permissionTemplates")
-    for categ in [cat for cat in settings.CATEGORIES if cat not in special_categories]:
-        new_json[categ] = util.sort_list_by_key(util.dict_to_list(old_json[categ], "key"), "key")
-    for k, v in old_json[settings.LANGUAGES_SETTINGS].items():
-        new_json[settings.LANGUAGES_SETTINGS] = new_json.get(settings.LANGUAGES_SETTINGS, None) or {}
-        new_json[settings.LANGUAGES_SETTINGS][k] = util.sort_list_by_key(util.dict_to_list(v, "key"), "key")
-    new_json[settings.LANGUAGES_SETTINGS] = util.dict_to_list(new_json[settings.LANGUAGES_SETTINGS], "language", "settings")
-    new_json[settings.DEVOPS_INTEGRATION] = util.dict_to_list(old_json[settings.DEVOPS_INTEGRATION], "key")
-    new_json["permissions"] = util.perms_to_list(old_json["permissions"])
-    for v in old_json["permissionTemplates"].values():
-        if "permissions" in v:
-            v["permissions"] = util.perms_to_list(v["permissions"])
-    new_json["permissionTemplates"] = util.dict_to_list(old_json["permissionTemplates"], "key")
-    return new_json